diff --git "a/wandb/run-20241226_145403-c7ovtvny/logs/debug-internal.log" "b/wandb/run-20241226_145403-c7ovtvny/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20241226_145403-c7ovtvny/logs/debug-internal.log" @@ -0,0 +1,39624 @@ +2024-12-26 14:54:03,048 INFO StreamThr :823089 [internal.py:wandb_internal():85] W&B internal server running at pid: 823089, started at: 2024-12-26 14:54:03.046903 +2024-12-26 14:54:03,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status +2024-12-26 14:54:03,058 INFO WriterThread:823089 [datastore.py:open_for_write():87] open: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/run-c7ovtvny.wandb +2024-12-26 14:54:03,059 DEBUG SenderThread:823089 [sender.py:send():379] send: header +2024-12-26 14:54:03,102 DEBUG SenderThread:823089 [sender.py:send():379] send: run +2024-12-26 14:54:03,325 INFO SenderThread:823089 [dir_watcher.py:__init__():211] watching files in: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files +2024-12-26 14:54:03,326 INFO SenderThread:823089 [sender.py:_start_run_threads():1188] run started: c7ovtvny with start time 1735224843.04944 +2024-12-26 14:54:03,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: check_version +2024-12-26 14:54:03,348 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: check_version +2024-12-26 14:54:03,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: run_start +2024-12-26 14:54:03,451 DEBUG HandlerThread:823089 [system_info.py:__init__():26] System info init +2024-12-26 14:54:03,452 DEBUG HandlerThread:823089 [system_info.py:__init__():41] System info init done +2024-12-26 14:54:03,452 INFO HandlerThread:823089 [system_monitor.py:start():194] Starting system monitor +2024-12-26 14:54:03,452 INFO SystemMonitor:823089 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-26 14:54:03,452 INFO HandlerThread:823089 [system_monitor.py:probe():214] Collecting system info +2024-12-26 14:54:03,453 INFO SystemMonitor:823089 [interfaces.py:start():188] Started cpu monitoring +2024-12-26 14:54:03,454 INFO SystemMonitor:823089 [interfaces.py:start():188] Started disk monitoring +2024-12-26 14:54:03,454 INFO SystemMonitor:823089 [interfaces.py:start():188] Started gpu monitoring +2024-12-26 14:54:03,455 INFO SystemMonitor:823089 [interfaces.py:start():188] Started memory monitoring +2024-12-26 14:54:03,456 INFO SystemMonitor:823089 [interfaces.py:start():188] Started network monitoring +2024-12-26 14:54:03,533 DEBUG HandlerThread:823089 [system_info.py:probe():152] Probing system +2024-12-26 14:54:03,537 DEBUG HandlerThread:823089 [system_info.py:_probe_git():137] Probing git +2024-12-26 14:54:03,555 DEBUG HandlerThread:823089 [system_info.py:_probe_git():145] Probing git done +2024-12-26 14:54:03,555 DEBUG HandlerThread:823089 [system_info.py:probe():200] Probing system done +2024-12-26 14:54:03,555 DEBUG HandlerThread:823089 [system_monitor.py:probe():223] {'os': 'Linux-6.8.0-48-generic-x86_64-with-glibc2.39', 'python': '3.10.0', 'heartbeatAt': '2024-12-26T14:54:03.533915', 'startedAt': '2024-12-26T14:54:03.029672', 'docker': None, 'cuda': None, 'args': ('--model_family', 'llama', '--apply_instruct_masks', '--token_scaled_loss', '--seq_parallel_size', '8', '--report_to', 'wandb', '--do_train', '--model_name_or_path', '/scratch3/workspace/ctpham_umass_edu-ft/models/Llama-3-8B-ProLong-512k-Base/', '--config_name', '/scratch3/workspace/ctpham_umass_edu-ft/models/Llama-3-8B-ProLong-512k-Base/', '--tokenizer_name', '/scratch3/workspace/ctpham_umass_edu-ft/models/Llama-3-8B-ProLong-512k-Base/', '--run_name', 'balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_', '--output_dir', '/scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_', '--config_overrides_json', '', '--gradient_accumulation_steps', '2', '--per_device_train_batch_size', '1', '--bf16', '--learning_rate', '1e-6', '--min_lr_ratio', '0.1', '--lr_scheduler_type', 'cosine', '--max_grad_norm', '1.0', '--adam_beta1', '0.9', '--adam_beta2', '0.95', '--weight_decay', '0.1', '--warmup_ratio', '0.05', '--optim', 'adamw_torch', '--logging_steps', '1', '--log_level', 'info', '--save_steps', '200', '--dataloader_num_workers', '1', '--disable_tqdm', 'true', '--use_fast_tokenizer', 'false', '--remove_unused_columns', 'false', '--ddp_find_unused_parameters', 'false', '--fsdp', 'auto_wrap offload', '--gradient_checkpointing', '--tokenized_mds_train', '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete', '--cuda_empty_cache', '--num_train_epochs', '1'), 'state': 'running', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'codePathLocal': 'finetune.py', 'codePath': 'prolong-final/finetune.py', 'git': {'remote': 'https://github.com/chtmp223/BookGen-dev.git', 'commit': '438e68ae65a707227d28baa427c5f6fdb0932c7b'}, 'email': 'ctpham@umass.edu', 'root': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev', 'host': 'gpu021', 'username': 'ctpham_umass_edu', 'executable': '/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10', 'cpu_count': 112, 'cpu_count_logical': 112, 'cpu_freq': {'current': 903.8299553571429, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 790.24, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1600.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1700.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.117, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 787.786, 'min': 800.0, 'max': 3800.0}, {'current': 1800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1000.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 892.512, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 790.711, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 790.69, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1300.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1100.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1782.468, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1100.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1700.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1000.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.039, 'min': 800.0, 'max': 3800.0}, {'current': 782.714, 'min': 800.0, 'max': 3800.0}, {'current': 783.793, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 794.544, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.441, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 438.487850189209, 'used': 16.265323638916016}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.3287200927734}} +2024-12-26 14:54:03,556 INFO HandlerThread:823089 [system_monitor.py:probe():224] Finished collecting system info +2024-12-26 14:54:03,556 INFO HandlerThread:823089 [system_monitor.py:probe():227] Publishing system info +2024-12-26 14:54:03,556 DEBUG HandlerThread:823089 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2024-12-26 14:54:04,327 INFO Thread-12 :823089 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/conda-environment.yaml +2024-12-26 14:54:11,037 DEBUG HandlerThread:823089 [system_info.py:_save_conda():224] Saving conda packages done +2024-12-26 14:54:11,041 INFO HandlerThread:823089 [system_monitor.py:probe():229] Finished publishing system info +2024-12-26 14:54:11,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:11,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 14:54:11,048 DEBUG SenderThread:823089 [sender.py:send():379] send: files +2024-12-26 14:54:11,048 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-26 14:54:11,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: python_packages +2024-12-26 14:54:11,325 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: python_packages +2024-12-26 14:54:11,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:54:11,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:11,328 DEBUG SenderThread:823089 [sender.py:send():379] send: telemetry +2024-12-26 14:54:11,328 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:54:11,335 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/conda-environment.yaml +2024-12-26 14:54:11,336 INFO Thread-12 :823089 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-metadata.json +2024-12-26 14:54:11,336 INFO Thread-12 :823089 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/requirements.txt +2024-12-26 14:54:11,422 INFO wandb-upload_0:823089 [upload_job.py:push():130] Uploaded file /tmp/tmpe1bywh49wandb/g9n92zls-wandb-metadata.json +2024-12-26 14:54:11,484 DEBUG SenderThread:823089 [sender.py:send():379] send: config +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: telemetry +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: telemetry +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:11,485 WARNING SenderThread:823089 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: telemetry +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: telemetry +2024-12-26 14:54:11,485 DEBUG SenderThread:823089 [sender.py:send():379] send: config +2024-12-26 14:54:12,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:12,336 INFO Thread-12 :823089 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:54:13,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:13,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:14,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:14,338 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:54:15,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:16,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:17,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:18,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:19,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:19,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:20,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:21,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:22,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:23,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:24,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:25,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:25,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:26,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:54:26,326 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:54:26,347 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:54:26,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:27,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:28,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:29,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:30,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:30,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:31,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:32,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:33,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:34,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:35,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:36,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:36,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:37,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:37,353 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/config.yaml +2024-12-26 14:54:38,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:39,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:40,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:41,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:54:41,326 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:54:41,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:41,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:42,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:43,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:44,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:45,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:46,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:47,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:47,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:48,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:49,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:50,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:51,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:52,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:52,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:53,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:54,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:55,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:56,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:54:56,327 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:54:56,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:56,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:54:56,725 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:56,726 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:56,726 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:56,726 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:56,726 DEBUG SenderThread:823089 [sender.py:send():379] send: metric +2024-12-26 14:54:56,726 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:54:56,727 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:54:56,729 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:54:57,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:57,377 INFO Thread-12 :823089 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:54:57,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:54:58,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:54:58,378 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:54:59,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:00,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:01,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:02,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:03,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:03,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:03,456 DEBUG SystemMonitor:823089 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-26 14:55:03,458 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:55:04,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:05,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:06,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:07,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:08,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:08,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:09,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:09,384 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/config.yaml +2024-12-26 14:55:10,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:11,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:55:11,327 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:55:11,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:12,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:13,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:14,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:14,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:15,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:16,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:17,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:18,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:19,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:20,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:20,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:21,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:22,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:23,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:24,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:25,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:25,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:26,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:55:26,329 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:55:26,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:27,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:28,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:28,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:55:28,854 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:55:28,854 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:55:28,857 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:55:29,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:29,404 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:55:30,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:30,405 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:55:30,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:31,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:32,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:33,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:33,461 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:55:34,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:35,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:36,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:36,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:37,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:38,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:39,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:40,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:41,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:55:41,330 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:55:41,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:41,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:42,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:43,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:44,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:45,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:46,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:47,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:47,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:48,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:49,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:50,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:51,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:52,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:52,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:53,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:54,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:55,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:56,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:55:56,337 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:55:56,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:57,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:55:57,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:55:57,561 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:55:57,562 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:55:57,564 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:55:57,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:58,429 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:55:58,430 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:55:58,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:55:59,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:00,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:01,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:02,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:03,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:03,464 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:56:03,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:04,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:05,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:06,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:07,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:08,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:09,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:09,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:10,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:11,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:56:11,333 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:56:11,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:12,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:13,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:14,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:14,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:15,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:16,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:17,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:18,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:19,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:20,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:20,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:21,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:22,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:23,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:24,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:25,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:25,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:26,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:56:26,334 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:56:26,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:56:26,590 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:56:26,591 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:56:26,594 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:56:26,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:27,460 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:56:27,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:28,461 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:56:28,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:29,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:30,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:30,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:31,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:32,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:33,468 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:56:33,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:34,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:35,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:36,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:36,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:37,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:38,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:39,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:40,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:41,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:56:41,333 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:56:41,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:41,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:42,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:43,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:44,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:45,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:46,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:47,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:47,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:48,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:49,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:50,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:51,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:52,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:52,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:53,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:54,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:55,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:56,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:56:56,335 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:56:56,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:56:56,752 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:56:56,752 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:56:56,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:56,754 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:56:57,494 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:56:57,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:57,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:56:58,495 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:56:58,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:56:59,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:00,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:01,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:02,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:03,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:03,472 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:57:03,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:04,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:05,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:06,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:07,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:08,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:09,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:09,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:10,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:11,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:57:11,335 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:57:11,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:12,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:13,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:14,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:14,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:15,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:16,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:17,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:18,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:19,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:20,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:20,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:21,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:22,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:23,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:24,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:25,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:26,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:57:26,081 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:57:26,081 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:57:26,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:26,086 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:57:26,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:57:26,500 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:57:26,529 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:57:26,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:27,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:28,531 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:57:28,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:29,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:30,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:31,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:31,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:32,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:33,476 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:57:33,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:34,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:35,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:36,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:37,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:37,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:38,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:39,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:40,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:41,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:57:41,496 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:57:41,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:42,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:42,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:43,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:44,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:45,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:46,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:47,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:48,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:48,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:49,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:50,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:51,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:52,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:53,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:53,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:54,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:55,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:55,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:57:55,974 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:57:55,975 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:57:55,977 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:57:56,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:57:56,495 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:57:56,553 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:57:56,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:57,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:58,555 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:57:58,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:57:58,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:57:59,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:00,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:01,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:02,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:03,480 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:58:03,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:04,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:04,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:05,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:06,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:07,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:08,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:09,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:09,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:10,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:11,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:58:11,495 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:58:11,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:12,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:13,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:14,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:14,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:15,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:16,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:17,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:18,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:19,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:20,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:20,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:21,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:22,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:23,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:24,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:25,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:25,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:58:25,740 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:58:25,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:58:25,743 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:58:26,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:26,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:58:26,495 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:58:26,583 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:58:26,583 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:58:27,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:28,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:29,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:30,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:30,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:31,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:32,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:33,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:33,484 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:58:34,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:35,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:36,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:36,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:37,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:38,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:39,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:40,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:41,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:41,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:58:41,495 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:58:41,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:42,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:43,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:44,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:45,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:46,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:47,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:47,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:48,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:49,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:50,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:51,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:52,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:53,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:53,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:54,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:55,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:56,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:56,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:58:56,499 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:58:57,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:58,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:58:58,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:58:59,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:00,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:00,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:59:00,240 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:59:00,240 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:59:00,245 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:59:00,614 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:59:01,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:02,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:02,616 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:59:03,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:03,488 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:59:04,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:04,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:05,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:06,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:07,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:08,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:09,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:09,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:10,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:11,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:11,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:59:11,499 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:59:12,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:13,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:14,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:14,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:15,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:16,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:17,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:18,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:19,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:20,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:20,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:21,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:22,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:23,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:24,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:25,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:26,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:26,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:59:26,500 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:59:26,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:27,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:28,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:29,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:30,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:31,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:31,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:59:31,664 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:59:31,665 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:59:31,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:31,668 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 14:59:32,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:32,648 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 14:59:32,648 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 14:59:33,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:33,493 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 14:59:34,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:35,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:36,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:37,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:37,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:38,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:39,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:40,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:41,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:41,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:59:41,500 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:59:42,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:42,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:43,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:44,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:45,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:46,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:47,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:48,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:48,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:49,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:50,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:51,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:52,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:53,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:54,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:54,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:55,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:56,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:56,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 14:59:56,509 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 14:59:57,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:58,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:59,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 14:59:59,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 14:59:59,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 14:59:59,723 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 14:59:59,723 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 14:59:59,724 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:00:00,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:00,672 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:00:00,672 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:00:01,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:02,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:03,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:03,497 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:00:04,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:05,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:05,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:06,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:07,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:08,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:09,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:10,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:10,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:11,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:11,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:00:11,505 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:00:12,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:13,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:14,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:15,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:15,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:16,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:17,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:18,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:19,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:20,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:21,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:21,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:22,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:23,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:24,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:25,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:26,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:26,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:00:26,511 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:00:26,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:27,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:28,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:29,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:30,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:30,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:00:30,810 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:00:30,811 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:00:30,812 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:00:31,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:31,697 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:00:32,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:32,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:32,698 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:00:33,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:33,501 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:00:34,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:35,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:36,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:37,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:38,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:38,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:39,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:40,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:41,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:41,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:00:41,507 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:00:42,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:43,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:43,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:44,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:45,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:46,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:47,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:48,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:49,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:49,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:50,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:51,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:52,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:53,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:54,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:54,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:00:55,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:56,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:56,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:00:56,507 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:00:57,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:58,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:59,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:00:59,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:00,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:00,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:01:00,625 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:01:00,625 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:01:00,627 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:01:00,722 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:01:01,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:02,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:02,724 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:01:03,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:03,505 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:01:04,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:05,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:05,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:06,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:07,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:08,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:09,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:10,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:10,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:11,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:11,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:01:11,507 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:01:12,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:13,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:14,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:15,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:15,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:16,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:17,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:18,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:19,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:20,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:21,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:21,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:22,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:23,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:24,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:25,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:26,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:26,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:01:26,507 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:01:26,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:27,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:28,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:29,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:30,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:31,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:32,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:32,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:32,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:01:32,737 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:01:32,737 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:01:32,738 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:01:32,750 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:01:33,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:33,509 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:01:34,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:34,752 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:01:35,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:36,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:37,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:37,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:38,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:39,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:40,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:41,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:41,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:01:41,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:01:42,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:42,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:43,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:44,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:45,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:46,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:47,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:48,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:48,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:49,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:50,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:51,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:52,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:53,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:54,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:54,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:01:55,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:56,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:56,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:01:56,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:01:57,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:58,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:59,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:01:59,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:00,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:01,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:02,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:03,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:03,515 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:02:04,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:04,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:02:04,849 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:02:04,849 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:02:04,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:04,851 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:02:05,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:05,777 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:02:06,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:06,778 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:02:07,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:08,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:09,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:10,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:10,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:11,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:11,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:02:11,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:02:12,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:13,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:14,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:15,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:15,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:16,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:17,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:18,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:19,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:20,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:21,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:21,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:22,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:23,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:24,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:25,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:26,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:26,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:02:26,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:02:26,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:27,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:28,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:29,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:30,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:31,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:32,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:32,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:33,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:33,519 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:02:34,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:34,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:02:34,570 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:02:34,570 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:02:34,572 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:02:34,807 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:02:35,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:36,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:36,808 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:02:37,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:37,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:38,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:39,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:40,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:41,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:41,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:02:41,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:02:42,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:42,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:43,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:44,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:45,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:46,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:47,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:48,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:48,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:49,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:50,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:51,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:52,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:53,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:53,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:54,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:55,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:56,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:56,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:02:56,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:02:57,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:58,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:02:58,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:02:59,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:00,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:01,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:02,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:03,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:03,523 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:03:04,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:04,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:05,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:05,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:03:05,537 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:03:05,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:03:05,539 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:03:05,835 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:03:06,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:06,836 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:03:07,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:08,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:09,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:09,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:10,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:11,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:11,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:03:11,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:03:12,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:13,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:14,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:14,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:15,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:16,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:17,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:18,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:19,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:20,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:20,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:21,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:22,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:23,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:24,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:25,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:25,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:26,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:26,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:03:26,509 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:03:27,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:28,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:29,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:30,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:30,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:31,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:32,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:33,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:33,527 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:03:34,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:35,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:35,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:03:35,514 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:03:35,514 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:03:35,515 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:03:35,860 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:03:36,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:36,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:36,861 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:03:37,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:38,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:39,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:40,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:41,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:41,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:03:41,509 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:03:41,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:42,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:43,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:44,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:45,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:46,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:47,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:47,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:48,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:49,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:50,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:51,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:52,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:52,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:53,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:54,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:55,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:56,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:56,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:03:56,509 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:03:57,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:57,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:03:58,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:03:59,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:00,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:01,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:02,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:03,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:03,531 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:04:03,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:04,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:05,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:06,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:07,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:08,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:08,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:09,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:10,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:10,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:04:10,854 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:04:10,854 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:04:10,856 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:04:10,892 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:04:12,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:04:12,011 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:04:12,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:12,894 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:04:13,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:14,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:14,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:15,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:16,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:17,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:18,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:19,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:20,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:20,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:21,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:22,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:23,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:24,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:25,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:26,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:26,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:27,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:04:27,009 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:04:27,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:28,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:29,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:30,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:31,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:31,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:32,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:33,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:33,536 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:04:34,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:35,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:36,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:36,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:37,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:38,014 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:39,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:40,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:41,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:42,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:04:42,009 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:04:42,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:42,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:42,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:04:42,284 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:04:42,284 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:04:42,286 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:04:42,919 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:04:42,920 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:04:43,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:44,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:45,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:46,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:47,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:48,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:48,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:49,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:50,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:51,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:52,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:53,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:54,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:04:54,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:55,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:56,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:57,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:04:57,010 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:04:57,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:58,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:59,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:04:59,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:00,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:01,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:02,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:03,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:03,539 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:05:04,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:04,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:05,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:06,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:07,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:08,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:09,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:10,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:10,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:11,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:12,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:05:12,011 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:05:12,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:13,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:14,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:14,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:05:14,489 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:05:14,490 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:05:14,492 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:05:14,960 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:05:15,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:15,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:16,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:16,962 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:05:17,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:18,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:19,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:20,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:21,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:21,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:22,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:23,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:24,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:25,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:26,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:27,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:27,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:05:27,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:27,195 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:05:28,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:29,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:30,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:31,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:32,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:32,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:33,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:33,542 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:05:34,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:35,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:36,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:37,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:38,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:38,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:39,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:40,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:41,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:42,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:05:42,106 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:05:42,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:43,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:43,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:44,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:44,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:05:44,687 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:05:44,687 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:05:44,689 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:05:44,985 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:05:45,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:46,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:46,987 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:05:47,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:48,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:49,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:49,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:50,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:51,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:52,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:53,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:54,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:55,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:05:55,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:56,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:57,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:05:57,106 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:05:57,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:58,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:05:59,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:00,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:00,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:01,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:02,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:03,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:03,546 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:06:04,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:05,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:05,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:06,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:07,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:08,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:09,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:10,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:11,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:11,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:12,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:06:12,110 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:06:12,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:13,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:14,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:15,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:15,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:06:15,338 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:06:15,338 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:06:15,339 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:06:16,017 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:06:16,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:16,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:17,018 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:06:17,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:18,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:19,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:20,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:21,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:22,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:22,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:23,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:24,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:25,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:26,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:27,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:06:27,110 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:06:27,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:27,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:28,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:29,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:30,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:31,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:32,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:33,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:33,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:33,551 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:06:34,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:35,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:36,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:37,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:38,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:39,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:39,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:40,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:41,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:42,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:06:42,112 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:06:42,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:43,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:44,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:44,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:45,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:46,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:46,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:06:46,314 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:06:46,314 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:06:46,315 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:06:47,042 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:06:47,043 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:06:47,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:48,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:49,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:50,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:50,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:51,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:52,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:53,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:54,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:55,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:56,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:06:56,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:57,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:06:57,112 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:06:57,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:58,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:06:59,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:00,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:01,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:01,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:02,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:03,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:03,555 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:07:04,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:05,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:06,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:06,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:07,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:08,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:09,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:10,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:11,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:12,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:07:12,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:07:12,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:12,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:13,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:14,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:15,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:15,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:07:15,688 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:07:15,688 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:07:15,690 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:07:16,071 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:07:16,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:17,072 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:07:17,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:18,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:18,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:19,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:20,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:21,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:22,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:23,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:24,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:24,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:25,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:26,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:27,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:07:27,179 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:07:27,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:28,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:29,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:29,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:30,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:31,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:32,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:33,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:33,559 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:07:34,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:34,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:35,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:36,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:37,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:38,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:39,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:40,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:40,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:41,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:42,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:07:42,185 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:07:42,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:43,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:44,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:44,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:07:44,380 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:07:44,380 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:07:44,382 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:07:45,095 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:07:45,096 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:07:45,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:45,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:46,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:47,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:48,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:49,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:50,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:51,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:51,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:52,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:53,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:54,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:55,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:56,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:57,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:07:57,186 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:07:57,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:57,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:07:58,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:07:59,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:00,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:01,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:02,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:03,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:03,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:03,563 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:08:04,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:05,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:06,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:07,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:08,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:09,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:09,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:10,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:11,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:12,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:08:12,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:08:12,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:13,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:14,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:14,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:15,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:08:15,170 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:08:15,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:08:15,172 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:08:15,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:16,129 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:08:16,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:17,130 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:08:17,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:18,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:19,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:20,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:20,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:21,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:22,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:23,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:24,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:25,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:25,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:26,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:27,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:08:27,189 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:08:27,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:28,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:29,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:30,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:30,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:31,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:32,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:33,567 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:08:33,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:34,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:35,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:35,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:36,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:37,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:38,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:39,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:40,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:41,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:41,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:42,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:08:42,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:08:42,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:43,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:44,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:45,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:46,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:46,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:47,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:47,837 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:08:47,838 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:08:47,838 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:08:47,840 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:08:48,154 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:08:48,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:49,156 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:08:49,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:50,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:51,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:51,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:52,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:53,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:54,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:55,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:56,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:57,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:08:57,194 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:08:57,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:08:57,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:58,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:08:59,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:00,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:01,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:02,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:02,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:03,571 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:09:03,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:04,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:05,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:06,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:07,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:08,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:08,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:09,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:10,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:11,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:12,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:09:12,194 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:09:12,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:13,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:13,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:14,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:15,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:16,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:17,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:18,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:18,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:09:18,731 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:09:18,731 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:09:18,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:18,732 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:09:19,186 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:09:19,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:20,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:21,188 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:09:21,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:22,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:23,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:24,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:24,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:25,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:26,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:27,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:09:27,194 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:09:27,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:28,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:29,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:29,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:30,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:31,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:32,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:33,575 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:09:33,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:34,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:34,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:35,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:36,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:37,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:38,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:39,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:40,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:40,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:41,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:42,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:09:42,194 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:09:42,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:43,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:44,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:45,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:45,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:46,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:47,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:48,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:09:48,175 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:09:48,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:09:48,176 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:09:48,217 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:09:48,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:49,218 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:09:49,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:50,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:51,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:51,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:52,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:53,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:54,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:55,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:56,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:09:56,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:57,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:09:57,194 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:09:57,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:58,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:09:59,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:00,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:01,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:01,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:02,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:03,579 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:10:03,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:04,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:05,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:06,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:06,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:07,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:08,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:09,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:10,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:11,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:12,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:10:12,199 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:10:12,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:12,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:13,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:14,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:15,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:16,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:17,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:18,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:18,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:10:18,289 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:10:18,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:10:18,291 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:10:18,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:19,247 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:10:19,248 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:10:19,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:20,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:21,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:22,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:23,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:23,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:24,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:25,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:26,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:27,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:10:27,195 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:10:27,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:28,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:28,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:29,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:30,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:31,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:32,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:33,583 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:10:33,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:33,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:34,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:35,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:36,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:37,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:38,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:39,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:39,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:40,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:41,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:42,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:10:42,195 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:10:42,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:43,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:44,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:44,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:45,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:46,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:47,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:48,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:10:48,632 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:10:48,632 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:10:48,633 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:10:49,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:49,279 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:10:49,279 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:10:49,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:49,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:50,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:51,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:52,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:53,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:54,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:55,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:10:55,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:56,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:57,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:10:57,195 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:10:57,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:58,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:10:59,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:00,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:00,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:01,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:02,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:03,587 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:11:03,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:04,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:05,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:05,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:06,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:07,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:08,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:09,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:10,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:11,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:11,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:12,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:11:12,208 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:11:12,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:13,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:14,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:15,789 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:16,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:16,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:17,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:17,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:11:17,923 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:11:17,923 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:11:17,925 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:11:18,314 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:11:18,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:19,315 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:11:19,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:20,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:21,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:21,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:22,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:23,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:24,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:25,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:26,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:27,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:11:27,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:11:27,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:27,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:28,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:29,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:30,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:31,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:32,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:33,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:33,591 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:11:33,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:34,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:35,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:36,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:37,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:38,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:38,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:39,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:40,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:41,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:42,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:11:42,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:11:42,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:43,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:43,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:44,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:45,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:46,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:47,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:48,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:49,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:49,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:11:49,616 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:11:49,617 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:11:49,619 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:11:50,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:50,346 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:11:51,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:51,347 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:11:52,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:53,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:54,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:54,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:11:55,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:56,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:57,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:57,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:11:57,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:11:58,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:59,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:11:59,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:00,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:01,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:02,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:03,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:03,595 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:12:04,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:04,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:05,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:06,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:07,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:08,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:09,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:10,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:10,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:11,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:12,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:12,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:12:12,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:12:13,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:14,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:15,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:15,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:16,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:17,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:18,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:18,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:12:18,802 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:12:18,803 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:12:18,804 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:12:19,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:19,373 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:12:19,373 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:12:20,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:20,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:21,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:22,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:23,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:24,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:25,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:26,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:26,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:27,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:12:27,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:12:27,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:28,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:29,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:30,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:31,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:31,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:32,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:33,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:33,599 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:12:34,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:35,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:36,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:36,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:37,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:38,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:39,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:40,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:41,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:42,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:12:42,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:12:42,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:42,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:43,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:44,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:45,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:46,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:47,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:48,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:48,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:48,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:12:48,697 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:12:48,697 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:12:48,698 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:12:49,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:49,396 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:12:49,396 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:12:50,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:51,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:52,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:53,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:54,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:54,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:12:55,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:56,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:57,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:12:57,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:12:57,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:58,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:59,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:12:59,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:00,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:01,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:02,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:03,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:03,604 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:13:04,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:04,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:05,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:06,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:07,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:08,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:09,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:10,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:10,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:11,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:12,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:13:12,198 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:13:12,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:13,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:14,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:15,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:15,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:16,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:17,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:18,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:19,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:13:19,341 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:13:19,341 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:13:19,343 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:13:19,430 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:13:19,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:20,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:20,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:21,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:22,432 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:13:22,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:23,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:24,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:25,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:26,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:26,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:27,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:13:27,198 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:13:27,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:28,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:29,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:30,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:31,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:31,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:32,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:33,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:33,608 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:13:34,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:35,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:36,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:36,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:37,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:38,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:39,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:40,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:41,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:42,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:13:42,215 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:13:42,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:42,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:43,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:44,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:45,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:46,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:47,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:47,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:13:47,924 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:13:47,925 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:13:47,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:47,927 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:13:48,459 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:13:48,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:49,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:50,461 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:13:50,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:51,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:52,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:53,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:53,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:54,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:55,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:56,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:57,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:13:57,203 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:13:57,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:58,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:13:58,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:13:59,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:00,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:01,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:02,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:03,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:03,612 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:14:03,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:04,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:05,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:06,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:07,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:08,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:09,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:09,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:10,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:11,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:12,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:14:12,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:14:12,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:13,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:14,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:14,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:15,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:15,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:14:15,964 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:14:15,964 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:14:15,966 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:14:16,482 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:14:16,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:17,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:18,484 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:14:18,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:19,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:20,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:20,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:21,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:22,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:23,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:24,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:25,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:26,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:26,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:27,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:14:27,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:14:27,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:28,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:29,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:30,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:31,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:31,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:32,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:33,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:33,616 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:14:34,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:35,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:36,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:36,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:37,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:38,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:39,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:40,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:41,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:42,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:14:42,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:14:42,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:42,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:43,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:44,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:45,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:46,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:47,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:48,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:48,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:49,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:14:49,231 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:14:49,231 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:14:49,233 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:14:49,515 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:14:49,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:50,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:51,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:52,518 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:14:52,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:53,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:53,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:54,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:55,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:56,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:57,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:14:57,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:14:57,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:58,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:14:58,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:14:59,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:00,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:01,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:02,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:03,620 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:15:03,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:03,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:04,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:05,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:06,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:07,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:08,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:09,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:09,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:10,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:11,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:12,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:15:12,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:15:12,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:13,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:14,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:15,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:15,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:16,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:17,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:18,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:19,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:20,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:20,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:15:20,515 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:15:20,515 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:15:20,517 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:15:20,543 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:15:20,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:21,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:22,544 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:15:22,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:23,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:24,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:25,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:26,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:26,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:27,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:15:27,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:15:27,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:28,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:29,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:30,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:31,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:31,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:32,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:33,624 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:15:33,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:34,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:35,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:36,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:36,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:37,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:38,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:39,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:40,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:41,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:42,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:15:42,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:15:42,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:42,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:43,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:44,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:45,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:46,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:47,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:48,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:48,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:49,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:50,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:51,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:51,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:15:51,725 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:15:51,726 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:15:51,727 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:15:52,578 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:15:52,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:53,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:54,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:54,579 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:15:54,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:55,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:56,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:57,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:15:57,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:15:57,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:58,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:15:59,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:15:59,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:00,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:01,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:02,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:03,627 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:16:03,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:04,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:04,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:05,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:06,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:07,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:08,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:09,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:10,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:10,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:11,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:12,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:16:12,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:16:12,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:13,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:14,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:15,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:16,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:16,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:17,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:18,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:19,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:20,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:21,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:21,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:22,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:23,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:24,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:16:24,073 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:16:24,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:16:24,075 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:16:24,607 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:16:24,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:25,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:26,609 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:16:26,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:27,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:27,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:16:27,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:16:27,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:28,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:29,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:30,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:31,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:32,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:32,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:33,631 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:16:33,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:34,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:35,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:36,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:37,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:37,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:38,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:39,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:40,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:41,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:42,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:16:42,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:16:42,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:43,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:43,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:44,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:45,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:46,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:47,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:48,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:48,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:49,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:50,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:51,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:52,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:53,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:53,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:54,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:16:54,028 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:16:54,028 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:16:54,030 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:16:54,636 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:16:54,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:55,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:56,637 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:16:56,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:57,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:16:57,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:16:57,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:58,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:16:59,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:16:59,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:00,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:01,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:02,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:03,635 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:17:03,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:04,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:04,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:05,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:06,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:07,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:08,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:09,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:10,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:10,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:11,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:12,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:17:12,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:17:12,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:13,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:14,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:15,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:15,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:16,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:17,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:18,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:19,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:20,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:21,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:21,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:22,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:23,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:23,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:17:23,823 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:17:23,824 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:17:23,825 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:17:24,664 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:17:24,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:25,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:26,665 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:17:26,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:26,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:27,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:17:27,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:17:27,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:28,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:29,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:30,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:31,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:32,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:32,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:33,640 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:17:33,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:34,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:35,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:36,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:37,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:38,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:38,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:39,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:40,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:41,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:42,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:17:42,208 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:17:42,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:43,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:43,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:44,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:45,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:46,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:47,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:48,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:49,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:49,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:50,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:51,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:52,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:53,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:54,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:17:54,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:55,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:56,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:57,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:17:57,208 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:17:57,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:58,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:17:58,012 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:17:58,012 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:17:58,013 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:17:58,691 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:17:58,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:17:59,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:00,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:00,693 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:18:00,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:01,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:02,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:03,644 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:18:03,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:04,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:05,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:05,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:06,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:07,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:08,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:09,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:10,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:11,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:11,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:12,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:18:12,208 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:18:12,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:13,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:14,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:15,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:16,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:16,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:17,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:18,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:19,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:20,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:21,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:21,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:22,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:23,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:24,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:25,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:26,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:26,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:27,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:18:27,106 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:18:27,107 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:18:27,108 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:18:27,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:18:27,522 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:18:27,718 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:18:27,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:28,719 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:18:28,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:29,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:30,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:31,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:31,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:32,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:33,648 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:18:33,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:34,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:35,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:36,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:37,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:37,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:38,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:39,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:40,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:41,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:42,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:18:42,257 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:18:42,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:43,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:43,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:44,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:45,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:46,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:47,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:48,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:48,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:49,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:50,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:51,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:52,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:53,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:53,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:54,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:55,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:56,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:57,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:18:57,258 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:18:57,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:18:57,682 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:18:57,683 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:18:57,684 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:18:57,746 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:18:58,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:18:58,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:18:59,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:00,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:00,748 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:19:01,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:02,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:03,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:03,652 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:19:04,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:04,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:05,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:06,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:07,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:08,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:09,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:10,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:10,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:11,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:12,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:12,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:19:12,259 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:19:13,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:14,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:15,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:15,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:16,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:17,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:18,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:19,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:20,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:20,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:21,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:22,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:23,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:24,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:25,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:25,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:26,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:27,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:27,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:19:27,263 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:19:28,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:28,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:19:28,570 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:19:28,570 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:19:28,572 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:19:28,773 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:19:29,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:30,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:30,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:30,775 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:19:31,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:32,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:33,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:33,656 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:19:34,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:35,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:35,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:36,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:37,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:38,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:39,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:40,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:41,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:41,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:42,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:19:42,261 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:19:42,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:43,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:44,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:45,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:46,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:46,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:47,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:48,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:49,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:50,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:51,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:51,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:52,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:53,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:54,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:55,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:56,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:56,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:19:57,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:19:57,178 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:19:57,179 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:19:57,180 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:19:57,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:19:57,603 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:19:57,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:57,805 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:19:58,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:19:58,806 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:19:59,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:00,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:01,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:01,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:02,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:03,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:03,660 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:20:04,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:05,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:06,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:07,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:07,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:08,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:09,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:10,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:11,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:12,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:20:12,328 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:20:12,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:12,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:13,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:14,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:15,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:16,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:17,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:17,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:18,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:19,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:20,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:21,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:22,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:23,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:23,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:24,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:25,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:26,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:27,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:20:27,334 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:20:27,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:28,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:28,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:29,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:20:29,108 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:20:29,108 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:20:29,109 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:20:29,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:29,831 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:20:30,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:30,832 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:20:31,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:32,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:33,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:33,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:33,664 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:20:34,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:35,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:36,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:37,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:38,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:39,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:39,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:40,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:41,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:42,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:20:42,334 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:20:42,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:43,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:44,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:44,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:45,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:46,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:47,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:48,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:49,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:49,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:50,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:51,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:52,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:53,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:54,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:20:54,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:55,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:56,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:57,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:20:57,336 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:20:57,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:58,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:58,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:20:58,934 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:20:58,935 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:20:58,937 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:20:59,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:20:59,863 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:20:59,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:00,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:00,864 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:21:01,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:02,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:03,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:03,669 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:21:04,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:05,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:05,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:06,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:07,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:08,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:09,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:10,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:11,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:11,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:12,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:21:12,336 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:21:12,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:13,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:14,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:15,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:16,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:16,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:17,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:18,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:19,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:20,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:21,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:22,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:22,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:23,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:24,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:25,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:26,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:27,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:21:27,340 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:21:27,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:28,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:28,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:29,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:21:29,220 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:21:29,220 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:21:29,221 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:21:29,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:29,890 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:21:30,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:30,891 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:21:31,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:32,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:33,673 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:21:33,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:33,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:34,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:35,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:36,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:37,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:38,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:39,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:39,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:40,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:41,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:42,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:21:42,340 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:21:42,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:43,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:44,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:44,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:45,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:46,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:47,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:48,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:49,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:50,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:50,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:51,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:52,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:53,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:54,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:55,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:21:55,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:56,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:57,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:21:57,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:21:57,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:58,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:59,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:21:59,183 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:21:59,183 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:21:59,184 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:21:59,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:21:59,918 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:22:00,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:00,919 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:22:01,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:01,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:02,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:03,677 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:22:03,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:04,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:05,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:06,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:06,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:07,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:08,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:09,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:10,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:11,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:12,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:22:12,343 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:22:12,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:12,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:13,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:14,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:15,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:16,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:17,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:17,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:18,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:19,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:20,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:21,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:22,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:23,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:23,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:24,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:25,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:26,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:27,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:22:27,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:22:27,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:28,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:28,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:29,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:30,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:31,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:22:31,657 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:22:31,657 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:22:31,659 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:22:31,951 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:22:32,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:32,952 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:22:33,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:33,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:33,681 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:22:34,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:35,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:36,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:37,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:38,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:39,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:39,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:40,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:41,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:42,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:42,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:22:42,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:22:43,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:44,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:45,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:45,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:46,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:47,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:48,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:49,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:50,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:50,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:51,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:52,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:53,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:54,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:55,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:56,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:56,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:22:57,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:57,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:22:57,352 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:22:58,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:22:59,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:00,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:01,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:01,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:23:01,570 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:23:01,571 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:23:01,572 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:23:01,981 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:23:02,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:02,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:02,982 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:23:03,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:03,685 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:23:04,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:05,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:06,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:07,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:07,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:08,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:09,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:10,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:11,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:12,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:12,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:23:12,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:23:13,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:13,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:14,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:15,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:16,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:17,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:18,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:18,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:19,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:20,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:21,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:22,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:23,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:24,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:24,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:25,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:26,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:27,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:27,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:23:27,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:23:28,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:29,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:29,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:30,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:31,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:31,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:23:31,994 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:23:31,994 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:23:31,996 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:23:32,009 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:23:32,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:33,010 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:23:33,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:33,690 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:23:34,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:34,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:35,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:36,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:37,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:38,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:39,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:40,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:40,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:41,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:42,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:42,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:23:42,353 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:23:43,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:44,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:45,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:45,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:46,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:47,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:48,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:49,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:50,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:51,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:51,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:52,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:53,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:54,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:55,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:56,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:56,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:23:57,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:57,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:23:57,359 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:23:58,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:23:59,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:00,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:01,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:01,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:01,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:24:01,900 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:24:01,900 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:24:01,901 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:24:02,040 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:24:02,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:03,041 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:24:03,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:03,693 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:24:04,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:05,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:06,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:06,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:07,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:08,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:09,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:10,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:11,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:12,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:12,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:24:12,356 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:24:12,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:13,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:14,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:15,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:16,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:17,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:17,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:18,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:19,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:20,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:21,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:22,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:23,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:23,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:24,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:25,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:26,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:27,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:24:27,376 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:24:27,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:28,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:28,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:29,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:30,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:31,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:32,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:33,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:24:33,028 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:24:33,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:24:33,030 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:24:33,070 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:24:33,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:33,698 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:24:33,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:34,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:35,072 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:24:35,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:36,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:37,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:38,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:39,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:39,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:40,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:41,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:42,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:24:42,360 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:24:42,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:43,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:44,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:44,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:45,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:46,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:47,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:48,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:49,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:49,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:50,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:51,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:52,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:53,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:54,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:55,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:55,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:24:56,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:57,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:24:57,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:24:57,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:58,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:24:59,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:00,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:00,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:01,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:02,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:02,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:25:02,592 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:25:02,593 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:25:02,594 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:25:03,103 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:25:03,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:03,702 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:25:04,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:05,104 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:25:05,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:05,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:06,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:07,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:08,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:09,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:10,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:10,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:11,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:12,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:25:12,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:25:12,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:13,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:14,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:15,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:16,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:16,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:17,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:18,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:19,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:20,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:21,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:22,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:22,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:23,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:24,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:25,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:26,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:27,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:25:27,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:25:27,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:28,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:28,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:29,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:30,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:31,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:32,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:33,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:33,705 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:25:34,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:25:34,522 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:25:34,523 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:25:34,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:34,525 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:25:34,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:35,131 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:25:35,131 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:25:35,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:36,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:37,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:38,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:39,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:39,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:40,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:41,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:42,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:25:42,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:25:42,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:43,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:44,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:45,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:45,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:46,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:47,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:48,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:49,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:50,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:50,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:51,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:52,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:53,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:54,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:55,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:56,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:25:56,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:57,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:25:57,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:25:57,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:58,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:25:59,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:00,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:01,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:01,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:02,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:03,713 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:26:03,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:04,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:26:04,073 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:26:04,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:26:04,074 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:26:04,163 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:26:04,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:05,164 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:26:05,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:06,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:07,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:07,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:08,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:09,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:10,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:11,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:12,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:26:12,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:26:12,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:12,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:13,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:14,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:15,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:16,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:17,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:17,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:18,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:19,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:20,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:21,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:22,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:23,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:23,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:24,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:25,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:26,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:27,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:26:27,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:26:27,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:28,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:29,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:29,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:30,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:31,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:32,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:33,721 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:26:33,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:34,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:35,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:26:35,289 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:26:35,290 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:26:35,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:35,292 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:26:35,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:36,190 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:26:36,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:37,191 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:26:37,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:38,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:39,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:40,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:40,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:41,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:42,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:26:42,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:26:42,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:43,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:44,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:45,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:46,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:46,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:47,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:48,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:49,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:50,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:51,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:51,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:52,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:53,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:54,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:55,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:56,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:57,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:26:57,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:26:57,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:26:57,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:58,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:26:59,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:00,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:01,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:02,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:02,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:03,728 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:27:03,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:04,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:05,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:27:05,008 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:27:05,008 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:27:05,009 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:27:05,216 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:27:05,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:06,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:07,218 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:27:07,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:08,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:08,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:09,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:10,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:11,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:12,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:27:12,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:27:12,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:13,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:14,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:14,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:15,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:16,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:18,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:19,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:19,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:20,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:21,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:22,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:23,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:24,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:25,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:25,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:26,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:27,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:27,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:27:27,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:27:28,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:29,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:30,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:30,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:31,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:32,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:33,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:33,728 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:27:34,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:35,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:36,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:27:36,036 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:27:36,036 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:27:36,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:36,042 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:27:36,249 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:27:36,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:37,250 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:27:37,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:38,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:39,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:40,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:41,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:41,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:42,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:27:42,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:27:42,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:43,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:44,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:45,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:46,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:47,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:47,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:48,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:49,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:50,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:51,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:52,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:52,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:53,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:54,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:55,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:56,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:57,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:27:57,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:27:57,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:58,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:27:58,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:27:59,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:00,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:01,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:02,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:03,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:03,731 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:28:04,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:04,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:05,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:06,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:07,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:08,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:08,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:28:08,891 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:28:08,892 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:28:08,897 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:28:09,286 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:28:09,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:09,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:10,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:11,288 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:28:11,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:12,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:28:12,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:28:12,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:13,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:14,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:15,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:15,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:16,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:17,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:18,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:19,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:20,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:20,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:21,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:22,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:23,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:24,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:25,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:26,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:26,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:27,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:28:27,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:28:27,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:28,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:29,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:30,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:31,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:31,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:32,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:33,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:33,738 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:28:34,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:35,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:36,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:28:36,508 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:28:36,509 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:28:36,510 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:28:36,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:37,310 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:28:37,311 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:28:37,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:37,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:38,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:39,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:40,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:41,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:42,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:28:42,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:28:42,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:42,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:43,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:44,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:45,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:46,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:47,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:48,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:48,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:49,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:50,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:51,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:52,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:53,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:53,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:54,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:55,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:56,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:57,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:28:57,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:28:57,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:58,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:28:59,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:28:59,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:00,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:01,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:02,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:03,739 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:29:03,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:04,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:05,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:05,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:06,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:07,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:08,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:08,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:29:08,957 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:29:08,958 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:29:08,959 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:29:09,345 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:29:09,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:10,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:10,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:11,347 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:29:11,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:12,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:29:12,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:29:12,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:13,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:14,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:15,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:16,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:16,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:17,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:18,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:19,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:20,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:21,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:21,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:22,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:23,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:24,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:25,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:26,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:26,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:27,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:29:27,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:29:27,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:28,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:29,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:30,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:31,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:32,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:32,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:33,750 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:29:33,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:34,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:35,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:36,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:37,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:37,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:38,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:29:38,881 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:29:38,881 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:29:38,883 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:29:39,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:39,382 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:29:39,383 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:29:40,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:41,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:42,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:42,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:29:42,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:29:43,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:43,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:44,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:45,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:46,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:47,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:48,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:49,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:49,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:50,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:51,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:52,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:53,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:54,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:54,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:29:55,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:56,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:57,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:57,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:29:57,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:29:58,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:29:59,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:00,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:00,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:01,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:02,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:03,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:03,750 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:30:04,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:05,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:06,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:06,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:06,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:30:06,996 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:30:06,997 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:30:06,998 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:30:07,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:07,414 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:30:08,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:09,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:09,415 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:30:10,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:11,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:11,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:12,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:30:12,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:30:12,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:13,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:14,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:15,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:16,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:17,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:17,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:18,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:19,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:20,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:21,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:22,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:22,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:23,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:24,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:25,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:26,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:27,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:30:27,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:30:27,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:28,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:28,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:29,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:30,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:31,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:32,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:33,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:33,753 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:30:34,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:34,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:30:34,634 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:30:34,634 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:30:34,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:34,639 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:30:35,435 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:30:35,436 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:30:35,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:36,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:37,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:38,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:39,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:39,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:40,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:41,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:42,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:30:42,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:30:42,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:43,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:44,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:45,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:45,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:46,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:47,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:48,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:49,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:50,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:50,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:51,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:52,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:53,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:54,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:55,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:56,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:56,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:30:57,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:30:57,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:30:57,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:58,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:30:59,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:00,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:01,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:01,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:02,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:03,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:03,755 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:31:04,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:31:04,209 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:31:04,210 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:31:04,211 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:31:04,462 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:31:04,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:05,463 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:31:05,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:06,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:07,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:07,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:08,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:09,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:10,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:11,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:12,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:31:12,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:31:12,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:12,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:13,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:14,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:15,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:16,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:17,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:17,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:18,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:19,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:20,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:21,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:22,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:23,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:23,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:24,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:25,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:26,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:27,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:31:27,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:31:27,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:28,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:29,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:29,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:30,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:31,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:32,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:33,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:33,764 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:31:34,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:35,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:31:35,458 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:31:35,459 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:31:35,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:35,461 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:31:35,491 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:31:35,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:36,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:37,493 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:31:37,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:38,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:39,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:40,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:40,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:41,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:42,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:31:42,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:31:42,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:43,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:44,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:45,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:46,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:46,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:47,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:48,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:49,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:50,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:51,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:51,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:52,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:53,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:54,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:55,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:56,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:31:56,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:57,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:31:57,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:31:57,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:58,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:31:59,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:00,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:01,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:01,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:02,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:03,764 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:32:03,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:04,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:05,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:06,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:32:06,741 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:32:06,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:32:06,746 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:32:07,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:07,523 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:32:07,524 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:32:07,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:08,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:09,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:10,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:11,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:12,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:12,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:32:12,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:32:12,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:13,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:14,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:15,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:16,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:17,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:17,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:18,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:19,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:20,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:21,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:22,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:22,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:23,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:24,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:25,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:26,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:27,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:27,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:32:27,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:32:28,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:28,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:29,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:30,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:31,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:32,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:33,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:33,770 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:32:34,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:34,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:35,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:36,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:36,423 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:32:36,424 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:32:36,425 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:32:36,427 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:32:36,557 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:32:37,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:37,558 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:32:38,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:39,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:39,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:40,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:41,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:42,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:42,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:32:42,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:32:43,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:44,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:45,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:45,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:46,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:47,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:48,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:49,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:50,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:50,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:51,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:52,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:53,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:54,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:55,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:55,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:32:56,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:57,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:57,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:32:57,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:32:58,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:32:59,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:00,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:01,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:01,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:02,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:03,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:03,771 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:33:04,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:05,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:06,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:07,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:33:07,159 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:33:07,160 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:33:07,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:07,161 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:33:07,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:07,588 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:33:07,588 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:33:08,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:09,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:10,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:11,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:12,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:12,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:33:12,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:33:12,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:13,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:14,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:15,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:16,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:17,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:17,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:18,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:19,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:20,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:21,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:22,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:22,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:23,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:24,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:25,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:26,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:27,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:27,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:33:27,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:33:28,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:28,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:29,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:30,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:31,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:32,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:33,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:33,773 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:33:34,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:34,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:35,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:36,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:36,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:33:36,672 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:33:36,672 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:33:36,674 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:33:37,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:37,619 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:33:37,620 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:33:38,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:39,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:39,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:40,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:41,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:42,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:42,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:33:42,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:33:43,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:44,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:45,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:45,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:46,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:47,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:48,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:49,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:50,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:50,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:51,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:52,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:53,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:54,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:55,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:55,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:33:56,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:57,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:57,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:33:57,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:33:58,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:33:59,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:00,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:00,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:01,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:02,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:03,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:03,780 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:34:04,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:05,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:06,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:06,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:34:06,501 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:34:06,502 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:34:06,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:06,503 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:34:06,642 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:34:07,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:07,643 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:34:08,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:09,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:10,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:11,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:11,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:12,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:12,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:34:12,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:34:13,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:14,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:15,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:16,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:17,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:17,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:18,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:19,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:20,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:21,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:22,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:22,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:23,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:24,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:25,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:26,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:27,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:27,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:34:27,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:34:28,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:28,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:29,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:30,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:31,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:32,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:33,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:33,796 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:34:34,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:34,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:35,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:36,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:37,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:38,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:39,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:39,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:34:39,661 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:34:39,662 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:34:39,667 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:34:39,679 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:34:40,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:40,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:41,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:41,681 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:34:42,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:42,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:34:42,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:34:43,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:44,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:45,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:45,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:46,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:47,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:48,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:49,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:50,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:51,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:51,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:52,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:53,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:54,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:55,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:56,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:57,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:57,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:34:57,368 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:34:57,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:34:58,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:34:59,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:00,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:01,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:02,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:02,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:03,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:03,789 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:35:04,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:05,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:06,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:07,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:07,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:08,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:09,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:10,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:11,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:11,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:35:11,773 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:35:11,773 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:35:11,779 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:35:12,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:12,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:35:12,456 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:35:12,711 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:35:13,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:13,711 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:35:13,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:14,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:15,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:16,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:17,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:18,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:18,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:19,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:20,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:21,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:22,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:23,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:24,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:24,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:25,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:26,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:27,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:35:27,452 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:35:27,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:28,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:29,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:30,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:30,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:31,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:32,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:33,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:33,793 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:35:34,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:35,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:36,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:36,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:37,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:38,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:39,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:40,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:41,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:41,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:42,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:35:42,452 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:35:42,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:43,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:35:43,394 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:35:43,394 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:35:43,396 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:35:43,743 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:35:43,744 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:35:43,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:44,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:45,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:46,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:47,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:47,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:48,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:49,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:50,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:51,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:52,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:52,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:53,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:54,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:55,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:56,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:57,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:35:57,452 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:35:57,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:57,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:35:58,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:35:59,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:00,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:01,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:02,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:02,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:03,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:03,797 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:36:04,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:05,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:06,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:07,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:07,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:08,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:09,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:10,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:11,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:12,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:36:12,452 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:36:12,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:36:12,580 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:36:12,581 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:36:12,582 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:36:12,769 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:36:12,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:12,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:13,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:13,776 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:36:14,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:15,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:16,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:17,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:18,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:18,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:19,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:20,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:21,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:22,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:23,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:24,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:24,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:25,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:26,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:27,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:36:27,611 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:36:27,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:28,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:29,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:30,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:30,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:31,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:32,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:33,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:33,801 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:36:34,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:35,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:36,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:36,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:37,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:38,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:39,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:40,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:41,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:41,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:42,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:36:42,527 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:36:42,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:43,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:44,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:36:44,330 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:36:44,331 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:36:44,336 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:36:44,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:44,804 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:36:45,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:46,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:46,806 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:36:47,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:47,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:48,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:49,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:50,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:51,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:52,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:52,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:53,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:54,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:55,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:56,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:57,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:36:57,567 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:36:57,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:57,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:36:58,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:36:59,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:00,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:01,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:02,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:02,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:03,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:03,806 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:37:04,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:05,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:06,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:07,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:08,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:08,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:09,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:10,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:11,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:12,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:37:12,577 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:37:12,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:13,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:13,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:14,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:15,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:16,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:37:16,285 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:37:16,286 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:37:16,289 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:37:16,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:16,840 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:37:17,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:18,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:18,841 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:37:19,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:19,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:20,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:21,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:22,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:23,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:24,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:25,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:25,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:26,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:27,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:37:27,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:37:27,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:28,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:29,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:30,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:31,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:31,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:32,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:33,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:33,808 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:37:34,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:35,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:36,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:37,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:37,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:38,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:39,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:40,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:41,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:42,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:37:42,579 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:37:42,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:42,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:43,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:44,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:45,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:46,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:47,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:47,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:48,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:37:48,190 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:37:48,191 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:37:48,196 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:37:48,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:48,870 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:37:49,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:50,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:50,872 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:37:51,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:52,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:52,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:53,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:54,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:55,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:56,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:57,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:37:57,576 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:37:57,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:57,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:37:58,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:37:59,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:00,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:01,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:02,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:03,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:03,812 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:38:03,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:04,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:05,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:06,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:07,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:08,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:08,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:09,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:10,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:11,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:12,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:38:12,576 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:38:12,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:13,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:13,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:14,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:15,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:16,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:17,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:38:17,179 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:38:17,179 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:38:17,180 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:38:17,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:17,897 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:38:18,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:18,898 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:38:19,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:19,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:20,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:21,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:22,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:23,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:24,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:25,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:25,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:26,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:27,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:38:27,576 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:38:27,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:28,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:29,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:30,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:31,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:31,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:32,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:33,816 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:38:33,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:34,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:35,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:36,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:37,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:37,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:38,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:39,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:40,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:41,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:42,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:38:42,590 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:38:42,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:42,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:43,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:44,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:45,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:46,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:47,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:48,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:38:48,132 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:38:48,132 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:38:48,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:48,133 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:38:48,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:48,928 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:38:49,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:50,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:50,930 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:38:51,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:52,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:53,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:53,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:38:54,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:55,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:56,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:57,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:38:57,580 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:38:57,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:58,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:59,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:38:59,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:00,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:01,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:02,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:03,821 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:39:03,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:04,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:05,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:05,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:06,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:07,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:08,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:09,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:10,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:10,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:11,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:12,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:39:12,589 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:39:12,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:13,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:14,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:15,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:15,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:16,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:39:16,723 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:39:16,723 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:39:16,725 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:39:16,954 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:39:17,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:17,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:18,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:18,956 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:39:19,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:20,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:20,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:21,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:22,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:23,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:24,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:25,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:25,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:26,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:27,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:39:27,583 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:39:27,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:28,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:29,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:30,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:31,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:31,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:32,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:33,825 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:39:33,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:34,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:35,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:36,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:37,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:37,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:38,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:39,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:40,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:41,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:42,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:39:42,587 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:39:42,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:42,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:43,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:44,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:45,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:46,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:39:46,386 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:39:46,387 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:39:46,388 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:39:46,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:46,984 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:39:47,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:48,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:48,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:48,986 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:39:49,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:50,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:51,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:52,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:53,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:53,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:39:54,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:55,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:56,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:57,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:39:57,587 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:39:57,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:58,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:59,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:39:59,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:00,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:01,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:02,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:03,830 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:40:03,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:04,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:05,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:05,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:06,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:07,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:08,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:09,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:10,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:10,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:11,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:12,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:40:12,592 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:40:12,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:13,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:14,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:15,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:15,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:16,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:17,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:18,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:40:18,048 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:40:18,048 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:40:18,050 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:40:18,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:19,014 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:40:19,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:20,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:21,015 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:40:21,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:21,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:22,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:23,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:24,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:25,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:26,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:26,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:27,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:40:27,590 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:40:27,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:28,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:29,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:30,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:31,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:32,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:32,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:33,833 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:40:33,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:34,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:35,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:36,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:37,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:38,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:38,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:39,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:40,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:41,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:42,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:40:42,592 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:40:42,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:43,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:43,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:44,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:45,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:40:45,497 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:40:45,498 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:40:45,500 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:40:45,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:46,045 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:40:46,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:47,046 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:40:47,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:48,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:49,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:49,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:50,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:51,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:52,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:53,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:54,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:54,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:40:55,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:56,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:57,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:40:57,592 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:40:57,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:58,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:40:59,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:00,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:00,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:01,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:02,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:03,837 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:41:03,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:04,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:05,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:06,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:06,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:07,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:08,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:09,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:10,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:11,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:11,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:12,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:41:12,595 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:41:12,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:13,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:14,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:15,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:16,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:16,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:17,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:41:17,067 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:41:17,068 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:41:17,070 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:41:17,071 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:41:17,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:18,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:19,073 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:41:19,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:20,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:21,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:22,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:22,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:23,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:24,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:25,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:26,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:27,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:41:27,595 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:41:27,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:28,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:28,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:29,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:30,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:31,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:32,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:33,841 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:41:33,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:34,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:34,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:35,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:36,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:37,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:38,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:39,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:39,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:40,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:41,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:42,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:41:42,611 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:41:42,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:43,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:44,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:44,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:45,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:46,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:47,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:48,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:41:48,192 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:41:48,193 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:41:48,195 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:41:48,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:49,115 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:41:49,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:50,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:50,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:51,120 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:41:51,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:52,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:53,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:54,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:55,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:55,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:41:56,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:57,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:41:57,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:41:57,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:58,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:41:59,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:00,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:01,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:01,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:02,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:03,845 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:42:03,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:04,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:05,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:06,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:07,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:07,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:08,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:09,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:10,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:11,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:12,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:42:12,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:42:12,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:12,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:13,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:14,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:15,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:16,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:17,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:17,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:18,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:19,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:42:19,436 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:42:19,436 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:42:19,438 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:42:19,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:20,146 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:42:20,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:21,147 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:42:21,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:22,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:23,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:23,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:24,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:25,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:26,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:27,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:42:27,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:42:27,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:28,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:29,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:29,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:30,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:31,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:32,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:33,851 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:42:33,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:34,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:35,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:35,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:36,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:37,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:38,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:39,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:40,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:40,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:41,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:42,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:42:42,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:42:42,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:43,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:44,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:45,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:45,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:46,961 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:47,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:48,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:49,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:50,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:50,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:51,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:52,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:42:52,938 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:42:52,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:42:52,940 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:42:53,177 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:42:53,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:54,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:55,178 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:42:55,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:56,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:56,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:42:57,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:57,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:42:57,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:42:58,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:42:59,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:00,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:01,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:02,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:02,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:03,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:03,854 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:43:04,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:05,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:06,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:07,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:07,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:08,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:09,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:10,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:11,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:12,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:12,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:43:12,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:43:13,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:13,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:14,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:15,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:16,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:17,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:18,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:18,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:19,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:20,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:21,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:22,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:43:22,287 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:43:22,287 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:43:22,288 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:43:22,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:23,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:23,207 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:43:23,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:24,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:25,209 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:43:25,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:26,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:27,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:27,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:43:27,601 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:43:28,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:29,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:29,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:30,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:31,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:32,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:33,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:33,858 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:43:34,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:34,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:35,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:36,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:37,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:38,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:39,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:40,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:40,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:41,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:42,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:43:42,601 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:43:42,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:43,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:44,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:45,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:46,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:46,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:47,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:48,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:49,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:50,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:51,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:51,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:52,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:53,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:53,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:43:53,850 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:43:53,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:43:53,851 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:43:54,238 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:43:54,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:55,239 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:43:55,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:56,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:56,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:43:57,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:43:57,601 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:43:57,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:58,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:43:59,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:00,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:01,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:02,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:02,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:03,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:03,862 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:44:04,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:05,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:06,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:07,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:07,865 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:08,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:09,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:10,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:11,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:12,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:44:12,601 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:44:12,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:13,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:13,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:14,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:15,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:16,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:17,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:18,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:18,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:19,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:20,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:44:20,472 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:44:20,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:44:20,474 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:44:20,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:21,263 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:44:21,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:22,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:23,264 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:44:23,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:24,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:24,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:25,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:26,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:27,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:44:27,601 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:44:27,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:28,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:29,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:30,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:30,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:32,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:33,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:33,866 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:44:34,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:35,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:35,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:36,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:37,128 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:38,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:39,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:40,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:41,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:41,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:42,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:42,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:44:42,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:44:43,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:44,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:45,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:46,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:46,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:47,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:47,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:44:47,453 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:44:47,453 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:44:47,454 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:44:48,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:48,290 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:44:49,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:49,291 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:44:50,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:51,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:52,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:52,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:53,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:54,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:55,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:56,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:57,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:57,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:44:57,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:44:57,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:44:58,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:44:59,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:00,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:01,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:02,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:03,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:03,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:03,870 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:45:04,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:05,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:06,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:07,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:08,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:08,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:09,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:10,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:11,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:12,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:12,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:45:12,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:45:13,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:13,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:14,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:15,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:16,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:17,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:18,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:18,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:45:18,672 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:45:18,672 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:45:18,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:18,673 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:45:19,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:19,318 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:45:20,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:21,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:21,320 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:45:22,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:23,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:24,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:24,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:25,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:26,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:27,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:27,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:45:27,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:45:28,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:29,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:30,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:30,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:31,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:32,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:33,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:33,873 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:45:34,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:35,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:35,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:36,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:37,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:38,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:39,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:40,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:41,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:41,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:42,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:42,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:45:42,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:45:43,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:44,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:45,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:46,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:46,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:47,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:48,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:49,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:50,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:51,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:51,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:51,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:45:51,532 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:45:51,533 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:45:51,534 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:45:52,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:52,348 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:45:53,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:53,349 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:45:54,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:55,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:56,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:57,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:45:57,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:57,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:45:57,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:45:58,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:45:59,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:00,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:01,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:02,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:03,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:03,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:03,878 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:46:04,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:05,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:06,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:07,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:08,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:08,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:09,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:10,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:11,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:12,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:12,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:46:12,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:46:13,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:14,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:14,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:15,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:16,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:17,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:18,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:19,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:19,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:20,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:21,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:22,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:22,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:46:22,796 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:46:22,797 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:46:22,798 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:46:23,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:23,372 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:46:24,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:25,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:25,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:25,374 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:46:26,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:27,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:27,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:46:27,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:46:28,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:29,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:30,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:31,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:31,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:32,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:33,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:33,882 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:46:34,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:35,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:36,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:36,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:37,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:38,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:39,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:40,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:41,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:42,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:42,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:42,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:46:42,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:46:43,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:44,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:45,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:46,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:47,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:48,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:48,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:49,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:50,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:51,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:52,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:53,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:53,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:54,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:55,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:55,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:46:55,573 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:46:55,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:46:55,575 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:46:56,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:56,398 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:46:57,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:57,399 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:46:57,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:46:57,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:46:58,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:46:58,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:46:59,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:00,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:01,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:02,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:03,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:03,886 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:47:03,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:04,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:05,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:06,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:07,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:08,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:09,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:09,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:10,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:11,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:12,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:12,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:47:12,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:47:13,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:14,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:15,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:15,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:16,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:17,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:18,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:19,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:20,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:20,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:21,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:22,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:23,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:24,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:24,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:47:24,429 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:47:24,429 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:47:24,431 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:47:25,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:25,431 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:47:25,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:26,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:27,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:27,432 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:47:27,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:47:27,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:47:28,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:29,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:30,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:31,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:31,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:32,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:33,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:33,889 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:47:34,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:35,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:36,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:36,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:37,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:38,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:39,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:40,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:41,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:42,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:42,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:42,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:47:42,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:47:43,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:44,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:45,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:46,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:47,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:48,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:48,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:49,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:50,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:51,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:52,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:53,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:53,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:54,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:55,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:56,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:56,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:47:56,445 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:47:56,445 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:47:56,446 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:47:56,463 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:47:57,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:57,464 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:47:57,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:47:57,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:47:58,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:47:59,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:47:59,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:00,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:01,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:02,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:03,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:03,894 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:48:04,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:04,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:05,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:06,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:07,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:08,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:09,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:10,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:10,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:11,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:12,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:12,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:48:12,606 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:48:13,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:14,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:15,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:16,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:16,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:17,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:18,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:19,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:20,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:21,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:21,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:22,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:23,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:24,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:25,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:26,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:26,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:27,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:48:27,254 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:48:27,254 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:48:27,256 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:48:27,495 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:48:27,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:48:27,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:27,710 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:48:28,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:29,496 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:48:29,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:30,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:31,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:31,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:32,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:33,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:33,896 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:48:34,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:35,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:36,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:36,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:37,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:38,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:39,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:40,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:41,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:42,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:42,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:48:42,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:48:42,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:43,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:44,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:45,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:46,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:47,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:48,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:48,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:49,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:50,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:51,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:52,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:53,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:53,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:54,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:55,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:48:55,715 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:48:55,716 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:48:55,717 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:48:55,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:56,522 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:48:56,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:57,523 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:48:57,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:48:57,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:48:57,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:58,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:48:59,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:48:59,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:00,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:01,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:02,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:03,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:03,900 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:49:04,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:04,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:05,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:06,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:07,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:08,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:09,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:10,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:10,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:11,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:12,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:49:12,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:49:12,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:13,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:14,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:15,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:16,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:16,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:17,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:18,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:19,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:20,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:21,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:21,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:22,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:23,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:24,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:25,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:26,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:49:26,267 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:49:26,267 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:49:26,269 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:49:26,549 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:49:26,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:27,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:27,550 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:49:27,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:49:27,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:49:27,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:28,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:29,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:30,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:31,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:32,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:32,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:33,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:33,904 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:49:34,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:35,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:36,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:37,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:37,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:38,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:39,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:40,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:41,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:42,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:49:42,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:49:42,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:43,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:43,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:44,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:45,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:46,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:47,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:48,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:48,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:49,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:50,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:51,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:52,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:53,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:49:53,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:54,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:55,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:56,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:49:56,660 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:49:56,660 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:49:56,661 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:49:57,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:57,577 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:49:57,578 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:49:57,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:49:57,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:49:58,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:59,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:49:59,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:00,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:01,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:02,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:03,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:03,908 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:50:04,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:04,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:05,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:06,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:07,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:08,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:09,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:10,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:10,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:11,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:12,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:12,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:50:12,709 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:50:13,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:14,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:15,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:16,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:16,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:17,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:18,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:19,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:20,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:21,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:21,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:22,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:23,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:24,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:25,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:26,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:26,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:27,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:27,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:50:27,708 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:50:28,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:28,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:50:28,805 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:50:28,806 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:50:28,808 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:50:29,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:29,604 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:50:29,605 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:50:30,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:31,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:31,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:32,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:33,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:33,913 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:50:34,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:35,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:36,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:36,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:37,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:38,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:39,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:40,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:41,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:42,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:42,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:42,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:50:42,709 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:50:43,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:44,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:45,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:46,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:47,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:47,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:48,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:49,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:50,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:51,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:52,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:53,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:53,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:54,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:55,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:56,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:57,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:57,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:50:57,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:50:58,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:58,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:50:58,987 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:50:58,988 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:50:58,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:50:58,989 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:50:59,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:50:59,636 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:50:59,636 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:51:00,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:01,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:02,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:03,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:03,916 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:51:04,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:04,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:05,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:06,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:07,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:08,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:09,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:10,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:10,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:11,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:12,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:12,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:51:12,712 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:51:13,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:14,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:15,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:15,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:16,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:17,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:18,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:19,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:20,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:21,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:21,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:22,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:23,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:24,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:25,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:26,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:26,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:27,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:27,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:51:27,716 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:51:28,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:29,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:30,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:31,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:31,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:32,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:33,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:51:33,130 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:51:33,131 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:51:33,132 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:51:33,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:33,675 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:51:33,676 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:51:33,921 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:51:34,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:35,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:36,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:36,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:37,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:38,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:39,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:40,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:41,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:42,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:42,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:42,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:51:42,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:51:43,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:44,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:45,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:46,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:47,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:48,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:48,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:49,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:50,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:51,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:52,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:53,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:53,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:54,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:55,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:56,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:57,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:57,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:51:57,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:51:58,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:51:58,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:51:59,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:00,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:01,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:02,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:03,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:03,923 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:52:03,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:04,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:52:04,026 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:52:04,026 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:52:04,028 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:52:04,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:04,707 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:52:05,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:05,708 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:52:06,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:07,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:08,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:09,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:09,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:10,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:11,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:12,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:12,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:52:12,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:52:13,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:14,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:14,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:15,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:16,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:17,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:18,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:19,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:20,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:20,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:21,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:22,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:23,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:24,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:25,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:26,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:26,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:27,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:27,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:52:27,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:52:28,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:29,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:30,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:31,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:31,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:32,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:33,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:33,929 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:52:34,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:52:34,388 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:52:34,389 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:52:34,390 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:52:34,744 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:52:34,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:35,745 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:52:35,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:36,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:37,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:37,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:38,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:39,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:40,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:41,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:42,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:42,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:52:42,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:52:42,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:43,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:44,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:45,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:46,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:47,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:48,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:48,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:49,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:50,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:51,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:52,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:53,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:54,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:54,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:55,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:56,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:57,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:52:57,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:52:57,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:58,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:52:59,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:52:59,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:00,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:01,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:02,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:03,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:03,934 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:53:04,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:04,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:04,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:53:04,973 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:53:04,974 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:53:04,976 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:53:05,780 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:53:05,781 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:53:05,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:06,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:07,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:08,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:09,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:10,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:10,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:11,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:12,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:53:12,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:53:12,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:13,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:14,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:15,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:16,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:16,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:17,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:18,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:19,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:20,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:21,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:21,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:22,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:23,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:24,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:25,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:26,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:27,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:27,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:53:27,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:53:27,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:28,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:29,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:30,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:31,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:32,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:32,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:33,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:33,941 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:53:34,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:35,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:53:35,006 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:53:35,007 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:53:35,008 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:53:35,807 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:53:35,807 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:53:35,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:36,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:37,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:38,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:38,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:39,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:40,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:41,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:42,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:53:42,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:53:42,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:43,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:43,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:44,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:45,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:46,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:47,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:48,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:49,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:49,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:50,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:51,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:52,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:53,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:54,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:53:54,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:55,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:56,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:57,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:53:57,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:53:57,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:58,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:53:59,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:00,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:00,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:01,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:02,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:03,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:03,941 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:54:04,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:05,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:05,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:06,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:07,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:54:07,322 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:54:07,322 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:54:07,324 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:54:07,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:07,844 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:54:07,845 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:54:08,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:09,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:10,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:11,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:11,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:12,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:54:12,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:54:12,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:13,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:14,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:15,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:16,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:17,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:17,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:18,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:19,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:20,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:21,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:22,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:22,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:23,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:24,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:25,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:26,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:27,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:54:27,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:54:27,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:27,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:28,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:29,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:30,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:31,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:32,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:33,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:33,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:33,947 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:54:34,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:35,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:36,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:54:36,174 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:54:36,174 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:54:36,175 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:54:36,876 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:54:36,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:37,877 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:54:37,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:38,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:39,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:39,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:40,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:41,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:42,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:54:42,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:54:42,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:43,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:44,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:45,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:45,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:46,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:47,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:48,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:49,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:50,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:50,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:51,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:52,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:53,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:54,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:55,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:56,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:54:56,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:57,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:54:57,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:54:57,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:58,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:54:59,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:00,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:01,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:02,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:02,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:03,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:03,953 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:55:04,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:05,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:06,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:07,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:55:07,398 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:55:07,399 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:55:07,400 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:55:07,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:07,908 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:55:07,909 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:55:08,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:08,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:09,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:10,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:11,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:12,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:55:12,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:55:12,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:13,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:13,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:14,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:15,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:16,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:17,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:18,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:19,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:19,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:20,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:21,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:22,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:23,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:24,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:25,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:25,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:26,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:27,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:55:27,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:55:27,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:28,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:29,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:30,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:31,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:31,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:32,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:33,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:33,953 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:55:34,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:35,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:36,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:55:36,888 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:55:36,888 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:55:36,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:36,892 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:55:36,940 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:55:37,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:37,941 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:55:38,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:39,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:40,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:41,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:42,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:42,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:42,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:55:42,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:55:43,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:44,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:45,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:46,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:47,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:48,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:48,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:49,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:50,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:51,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:52,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:53,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:54,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:54,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:55:55,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:56,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:57,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:57,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:55:57,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:55:58,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:59,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:55:59,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:00,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:01,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:02,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:03,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:03,953 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:56:04,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:04,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:05,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:06,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:07,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:07,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:56:07,968 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:56:07,968 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:56:07,971 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:56:07,971 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:56:08,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:08,971 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:56:09,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:09,972 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:56:10,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:10,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:11,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:12,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:12,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:56:12,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:56:13,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:14,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:15,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:16,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:16,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:17,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:18,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:19,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:20,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:21,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:21,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:22,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:23,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:24,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:25,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:26,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:26,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:27,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:27,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:56:27,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:56:28,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:29,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:30,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:31,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:32,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:32,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:33,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:33,961 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:56:34,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:35,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:36,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:37,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:37,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:38,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:56:38,271 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:56:38,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:56:38,276 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:56:38,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:38,998 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:56:39,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:39,999 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:56:40,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:41,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:42,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:42,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:56:42,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:56:43,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:43,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:44,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:45,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:46,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:47,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:48,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:48,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:49,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:50,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:51,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:52,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:53,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:53,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:54,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:55,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:56,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:57,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:56:57,719 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:56:57,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:58,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:56:59,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:56:59,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:00,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:01,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:02,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:03,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:03,965 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:57:04,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:04,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:05,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:06,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:07,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:57:07,141 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:57:07,141 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:57:07,143 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:57:07,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:08,022 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:57:08,023 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:57:08,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:09,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:10,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:10,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:11,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:12,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:57:12,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:57:12,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:13,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:14,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:15,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:16,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:16,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:17,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:18,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:19,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:20,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:21,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:21,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:22,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:23,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:24,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:25,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:26,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:27,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:57:27,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:57:27,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:27,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:28,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:29,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:30,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:31,837 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:32,837 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:32,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:33,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:33,967 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:57:34,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:35,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:36,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:57:36,375 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:57:36,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:57:36,377 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:57:36,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:37,051 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:57:37,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:38,052 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:57:38,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:38,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:39,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:40,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:41,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:42,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:57:42,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:57:42,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:43,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:43,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:44,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:45,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:46,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:47,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:48,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:48,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:50,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:51,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:52,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:53,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:53,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:54,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:55,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:56,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:57,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:57,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:57:57,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:57:58,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:57:58,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:57:59,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:00,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:01,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:02,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:03,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:03,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:03,968 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:58:04,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:05,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:06,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:07,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:07,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:58:07,871 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:58:07,871 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:58:07,876 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:58:08,077 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:58:08,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:08,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:09,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:10,079 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:58:10,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:11,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:12,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:12,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:58:12,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:58:13,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:14,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:14,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:15,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:16,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:17,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:18,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:19,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:19,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:20,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:21,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:22,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:23,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:24,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:24,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:25,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:26,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:27,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:27,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:58:27,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:58:28,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:29,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:30,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:30,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:31,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:32,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:33,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:33,979 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:58:34,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:35,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:35,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:36,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:37,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:38,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:38,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:58:38,747 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:58:38,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:58:38,750 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:58:39,108 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:58:39,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:40,109 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:58:40,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:41,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:41,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:42,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:42,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:58:42,721 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:58:43,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:44,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:45,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:46,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:47,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:47,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:48,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:49,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:50,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:51,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:52,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:53,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:53,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:54,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:55,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:56,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:57,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:57,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:58:57,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:58:58,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:58:58,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:58:59,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:00,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:01,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:02,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:03,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:03,985 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:59:03,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:04,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:05,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:06,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:07,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:08,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:08,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:59:08,959 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:59:08,960 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:59:08,965 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:59:08,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:09,137 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:59:09,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:10,138 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:59:10,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:11,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:12,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:12,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:59:12,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:59:13,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:14,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:14,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:15,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:16,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:17,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:18,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:19,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:20,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:20,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:21,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:22,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:23,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:24,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:25,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:25,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:26,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:27,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:27,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:59:27,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:59:28,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:29,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:30,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:31,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:31,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:32,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:33,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:33,986 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 15:59:34,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:35,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:36,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:36,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:37,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:38,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:39,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 15:59:39,224 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 15:59:39,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 15:59:39,229 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 15:59:39,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:40,163 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 15:59:40,164 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 15:59:40,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:41,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:41,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:42,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:42,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:59:42,721 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:59:43,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:44,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:45,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:46,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:47,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:47,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:48,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:49,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:50,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:51,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:52,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:53,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:53,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:54,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:55,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:56,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:57,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:57,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 15:59:57,721 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 15:59:58,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 15:59:58,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 15:59:59,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:00,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:01,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:02,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:03,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:03,988 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:00:03,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:04,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:05,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:06,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:07,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:08,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:09,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:09,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:10,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:11,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:12,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:00:12,248 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:00:12,248 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:00:12,251 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:00:12,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:12,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:00:12,721 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:00:13,196 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:00:13,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:14,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:15,197 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:00:15,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:15,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:16,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:17,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:18,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:19,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:20,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:21,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:21,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:22,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:23,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:24,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:25,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:26,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:27,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:27,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:27,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:00:27,721 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:00:28,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:29,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:30,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:31,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:32,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:32,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:33,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:33,991 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:00:34,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:35,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:36,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:37,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:38,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:38,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:39,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:40,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:41,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:42,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:42,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:00:42,723 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:00:43,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:43,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:44,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:00:44,056 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:00:44,057 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:00:44,060 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:00:44,227 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:00:44,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:45,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:46,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:47,229 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:00:47,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:48,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:49,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:49,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:50,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:51,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:52,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:53,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:54,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:55,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:00:55,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:56,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:57,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:57,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:00:57,724 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:00:58,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:00:59,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:00,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:00,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:01,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:02,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:03,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:03,995 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:01:04,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:05,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:06,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:06,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:07,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:08,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:09,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:10,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:11,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:11,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:12,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:12,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:01:12,727 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:01:13,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:14,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:14,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:01:14,832 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:01:14,833 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:01:14,835 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:01:15,254 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:01:15,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:16,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:16,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:17,256 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:01:17,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:18,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:19,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:20,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:21,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:22,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:22,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:23,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:24,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:25,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:26,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:27,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:27,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:01:27,727 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:01:27,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:28,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:29,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:30,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:31,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:32,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:33,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:33,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:34,000 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:01:34,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:35,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:36,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:37,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:38,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:38,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:39,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:40,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:41,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:42,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:01:42,729 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:01:42,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:43,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:01:43,242 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:01:43,243 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:01:43,244 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:01:43,284 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:01:43,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:44,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:44,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:45,285 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:01:45,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:46,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:47,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:48,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:49,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:49,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:50,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:51,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:52,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:53,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:54,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:55,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:01:55,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:56,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:57,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:01:57,729 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:01:57,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:58,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:01:59,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:00,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:00,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:01,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:02,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:03,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:04,004 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:02:04,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:05,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:06,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:06,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:07,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:08,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:09,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:10,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:11,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:11,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:12,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:02:12,731 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:02:12,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:13,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:14,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:02:14,091 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:02:14,091 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:02:14,092 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:02:14,321 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:02:14,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:15,322 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:02:15,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:16,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:16,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:17,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:18,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:19,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:20,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:21,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:21,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:22,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:23,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:24,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:25,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:26,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:27,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:27,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:02:27,731 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:02:27,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:28,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:29,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:30,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:31,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:32,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:32,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:33,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:34,007 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:02:34,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:35,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:36,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:37,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:37,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:38,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:39,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:40,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:41,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:42,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:42,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:02:42,738 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:02:42,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:43,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:02:43,738 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:02:43,738 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:02:43,740 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:02:44,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:44,351 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:02:44,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:45,352 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:02:45,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:46,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:47,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:48,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:48,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:49,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:50,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:51,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:52,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:53,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:54,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:02:54,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:55,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:56,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:57,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:02:57,735 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:02:57,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:58,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:02:59,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:00,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:00,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:01,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:02,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:03,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:04,011 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:03:04,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:05,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:06,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:06,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:07,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:08,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:09,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:10,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:11,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:11,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:12,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:03:12,737 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:03:12,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:13,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:03:13,384 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:03:13,384 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:03:13,386 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:03:13,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:14,383 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:03:14,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:15,384 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:03:15,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:16,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:17,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:17,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:18,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:19,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:20,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:21,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:22,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:22,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:23,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:24,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:25,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:26,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:27,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:03:27,737 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:03:27,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:27,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:28,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:29,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:30,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:31,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:32,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:33,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:33,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:34,015 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:03:34,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:35,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:36,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:37,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:38,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:38,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:39,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:40,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:41,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:42,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:03:42,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:03:42,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:43,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:43,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:44,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:45,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:03:45,329 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:03:45,330 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:03:45,331 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:03:45,416 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:03:45,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:46,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:47,418 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:03:47,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:48,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:49,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:49,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:50,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:51,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:52,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:53,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:54,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:54,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:55,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:56,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:57,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:03:57,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:03:57,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:58,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:03:59,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:03:59,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:00,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:01,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:02,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:03,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:04,021 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:04:04,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:05,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:05,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:06,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:07,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:08,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:09,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:10,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:10,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:11,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:12,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:04:12,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:04:12,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:13,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:14,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:15,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:15,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:16,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:04:16,029 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:04:16,030 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:04:16,031 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:04:16,447 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:04:16,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:17,447 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:04:17,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:18,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:19,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:20,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:21,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:21,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:22,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:23,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:24,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:25,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:26,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:27,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:27,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:04:27,745 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:04:27,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:28,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:29,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:30,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:31,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:32,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:32,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:33,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:34,024 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:04:34,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:35,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:36,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:37,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:37,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:38,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:39,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:40,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:41,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:42,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:42,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:04:42,745 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:04:42,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:43,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:04:43,102 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:04:43,102 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:04:43,103 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:04:43,479 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:04:43,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:44,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:45,481 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:04:45,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:46,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:47,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:48,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:48,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:49,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:50,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:51,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:52,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:53,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:54,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:54,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:55,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:56,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:57,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:04:57,745 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:04:57,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:58,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:04:59,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:04:59,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:00,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:01,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:02,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:03,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:04,028 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:05:04,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:05,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:05,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:06,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:07,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:08,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:09,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:10,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:10,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:11,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:12,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:05:12,748 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:05:12,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:13,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:14,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:15,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:16,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:05:16,283 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:05:16,283 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:05:16,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:16,284 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:05:16,507 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:05:16,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:17,507 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:05:17,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:18,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:19,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:20,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:21,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:21,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:22,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:23,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:24,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:25,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:26,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:26,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:27,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:05:27,748 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:05:27,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:28,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:29,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:30,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:31,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:31,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:32,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:33,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:34,032 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:05:34,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:35,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:36,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:37,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:37,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:38,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:39,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:40,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:41,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:42,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:42,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:05:42,753 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:05:42,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:43,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:44,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:45,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:46,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:46,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:05:46,998 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:05:46,999 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:05:47,000 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:05:47,535 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:05:47,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:48,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:48,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:49,536 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:05:49,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:50,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:51,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:52,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:53,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:53,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:54,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:55,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:56,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:57,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:05:57,753 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:05:57,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:58,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:05:58,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:05:59,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:00,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:02,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:03,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:04,036 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:06:04,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:04,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:05,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:06,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:07,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:08,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:09,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:09,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:10,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:11,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:12,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:12,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:06:12,753 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:06:13,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:14,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:15,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:15,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:16,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:17,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:18,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:19,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:19,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:06:19,423 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:06:19,424 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:06:19,425 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:06:19,564 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:06:20,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:20,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:21,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:21,566 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:06:22,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:23,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:24,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:25,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:26,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:26,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:27,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:27,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:06:27,753 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:06:28,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:29,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:30,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:31,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:31,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:32,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:33,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:34,040 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:06:34,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:35,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:36,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:37,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:37,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:38,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:39,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:40,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:41,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:42,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:42,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:06:42,754 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:06:42,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:43,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:44,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:45,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:46,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:47,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:48,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:48,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:49,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:49,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:06:49,500 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:06:49,501 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:06:49,503 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:06:49,597 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:06:50,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:51,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:51,599 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:06:52,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:53,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:53,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:06:54,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:55,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:56,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:57,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:57,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:06:57,753 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:06:58,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:59,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:06:59,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:00,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:01,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:02,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:03,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:04,046 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:07:04,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:04,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:05,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:06,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:07,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:08,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:09,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:10,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:10,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:11,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:12,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:12,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:07:12,760 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:07:13,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:14,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:15,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:15,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:16,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:17,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:17,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:07:17,445 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:07:17,445 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:07:17,447 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:07:17,624 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:07:18,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:19,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:19,625 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:07:20,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:21,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:21,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:22,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:23,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:24,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:25,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:26,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:27,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:27,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:07:27,760 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:07:27,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:28,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:29,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:30,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:31,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:32,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:32,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:33,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:34,049 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:07:34,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:35,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:36,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:37,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:38,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:38,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:39,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:40,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:41,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:42,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:42,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:07:42,783 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:07:43,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:43,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:44,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:45,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:46,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:47,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:48,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:48,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:07:48,534 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:07:48,535 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:07:48,536 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:07:48,660 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:07:49,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:49,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:49,661 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:07:50,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:51,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:52,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:53,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:54,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:54,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:07:55,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:56,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:57,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:57,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:07:57,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:07:58,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:07:59,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:00,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:00,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:01,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:02,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:03,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:04,054 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:08:04,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:05,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:06,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:06,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:07,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:08,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:09,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:10,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:11,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:11,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:12,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:12,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:08:12,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:08:13,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:14,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:15,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:16,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:16,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:17,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:18,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:19,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:20,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:20,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:08:20,916 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:08:20,916 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:08:20,917 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:08:21,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:21,690 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:08:21,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:22,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:23,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:23,692 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:08:24,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:25,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:26,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:27,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:27,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:08:27,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:08:27,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:28,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:29,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:30,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:31,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:32,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:32,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:33,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:34,058 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:08:34,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:35,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:36,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:37,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:37,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:38,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:39,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:40,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:41,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:42,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:42,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:08:42,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:08:43,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:43,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:44,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:45,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:46,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:47,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:48,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:48,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:49,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:50,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:51,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:52,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:52,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:08:52,822 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:08:52,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:08:52,824 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:08:53,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:53,721 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:08:54,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:54,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:08:55,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:55,722 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:08:56,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:57,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:57,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:08:57,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:08:58,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:59,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:08:59,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:00,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:01,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:02,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:03,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:04,060 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:09:04,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:04,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:05,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:06,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:07,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:08,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:09,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:09,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:10,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:11,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:12,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:12,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:09:12,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:09:13,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:14,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:15,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:15,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:16,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:17,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:18,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:19,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:20,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:20,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:21,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:21,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:09:21,860 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:09:21,860 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:09:21,862 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:09:22,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:22,752 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:09:23,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:23,753 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:09:24,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:25,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:26,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:26,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:27,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:27,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:09:27,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:09:28,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:29,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:30,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:31,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:32,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:32,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:33,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:34,065 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:09:34,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:35,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:36,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:37,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:37,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:38,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:39,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:40,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:41,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:42,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:42,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:09:42,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:09:43,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:43,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:44,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:45,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:46,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:47,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:48,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:48,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:49,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:50,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:51,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:52,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:09:52,495 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:09:52,496 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:09:52,497 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:09:52,780 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:09:52,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:53,781 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:09:53,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:54,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:54,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:55,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:56,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:57,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:09:57,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:09:57,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:58,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:09:59,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:09:59,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:00,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:01,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:02,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:03,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:04,069 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:10:04,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:04,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:05,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:06,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:07,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:08,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:09,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:09,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:10,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:11,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:12,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:10:12,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:10:12,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:13,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:14,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:15,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:15,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:16,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:17,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:18,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:19,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:20,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:20,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:21,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:22,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:23,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:10:23,368 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:10:23,368 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:10:23,370 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:10:23,809 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:10:23,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:24,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:25,812 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:10:25,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:26,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:26,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:27,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:10:27,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:10:27,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:28,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:29,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:30,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:31,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:32,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:32,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:33,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:34,073 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:10:34,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:35,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:36,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:37,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:38,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:38,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:39,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:40,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:41,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:42,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:10:42,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:10:42,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:43,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:44,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:44,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:45,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:46,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:47,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:48,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:49,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:49,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:50,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:51,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:52,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:53,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:54,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:54,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:10:55,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:56,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:10:56,299 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:10:56,300 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:10:56,301 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:10:56,840 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:10:56,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:57,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:10:57,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:10:57,841 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:10:57,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:58,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:10:59,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:00,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:00,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:01,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:02,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:03,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:04,077 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:11:04,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:05,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:06,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:06,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:08,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:09,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:10,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:11,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:11,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:12,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:12,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:11:12,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:11:13,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:14,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:15,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:16,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:17,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:17,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:18,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:19,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:20,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:21,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:22,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:22,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:23,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:24,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:25,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:25,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:11:25,511 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:11:25,511 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:11:25,512 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:11:25,869 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:11:26,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:27,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:27,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:11:27,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:11:27,871 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:11:28,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:28,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:29,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:30,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:31,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:32,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:33,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:33,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:34,081 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:11:34,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:35,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:36,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:37,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:38,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:39,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:39,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:40,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:41,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:42,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:42,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:11:42,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:11:43,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:44,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:44,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:45,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:46,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:47,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:48,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:49,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:50,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:50,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:51,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:52,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:53,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:54,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:55,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:55,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:11:56,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:56,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:11:56,188 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:11:56,188 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:11:56,190 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:11:56,900 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:11:57,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:57,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:11:57,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:11:57,901 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:11:58,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:11:59,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:00,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:01,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:01,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:02,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:03,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:04,085 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:12:04,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:05,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:06,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:07,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:07,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:08,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:09,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:10,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:11,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:12,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:12,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:12:12,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:12:13,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:13,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:14,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:15,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:16,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:17,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:18,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:18,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:19,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:20,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:21,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:22,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:23,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:23,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:24,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:25,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:26,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:26,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:12:26,803 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:12:26,803 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:12:26,804 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:12:26,926 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:12:27,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:27,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:12:27,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:12:27,927 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:12:28,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:29,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:29,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:30,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:31,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:32,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:33,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:34,090 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:12:34,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:34,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:35,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:36,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:37,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:38,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:39,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:39,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:40,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:41,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:42,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:42,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:12:42,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:12:43,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:44,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:45,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:45,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:46,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:47,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:48,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:49,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:50,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:51,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:51,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:52,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:53,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:54,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:55,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:56,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:56,961 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:12:57,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:12:57,224 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:12:57,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:12:57,227 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:12:57,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:57,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:12:57,768 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:12:57,953 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:12:58,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:59,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:12:59,955 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:13:00,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:01,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:02,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:02,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:03,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:04,093 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:13:04,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:05,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:06,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:07,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:08,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:08,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:09,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:10,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:11,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:12,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:12,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:13:12,767 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:13:13,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:14,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:14,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:15,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:16,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:17,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:18,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:19,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:19,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:20,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:21,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:22,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:23,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:24,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:24,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:25,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:26,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:26,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:13:26,527 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:13:26,527 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:13:26,529 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:13:26,982 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:13:27,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:27,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:13:27,768 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:13:27,983 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:13:28,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:29,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:30,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:30,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:31,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:32,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:33,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:34,097 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:13:34,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:35,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:36,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:36,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:37,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:38,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:39,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:40,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:41,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:41,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:42,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:42,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:13:42,768 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:13:43,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:44,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:45,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:46,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:47,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:47,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:48,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:49,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:50,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:51,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:52,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:52,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:53,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:54,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:55,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:56,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:56,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:13:56,488 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:13:56,489 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:13:56,490 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:13:57,010 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:13:57,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:57,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:13:57,768 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:13:58,012 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:13:58,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:13:58,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:13:59,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:00,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:01,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:02,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:03,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:03,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:04,102 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:14:04,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:05,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:06,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:07,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:08,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:08,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:09,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:10,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:11,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:12,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:12,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:14:12,768 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:14:13,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:14,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:14,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:15,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:16,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:17,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:18,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:19,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:19,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:20,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:21,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:22,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:23,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:24,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:24,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:25,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:26,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:27,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:27,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:14:27,420 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:14:27,421 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:14:27,422 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:14:27,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:14:27,840 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:14:28,039 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:14:28,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:29,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:30,041 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:14:30,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:30,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:31,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:32,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:33,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:34,106 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:14:34,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:35,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:36,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:36,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:37,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:38,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:39,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:40,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:41,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:41,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:42,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:42,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:14:42,840 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:14:43,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:44,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:45,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:46,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:47,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:47,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:48,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:49,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:50,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:51,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:52,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:52,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:53,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:54,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:55,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:55,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:14:55,958 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:14:55,958 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:14:55,960 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:14:56,068 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:14:56,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:57,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:57,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:14:57,840 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:14:58,070 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:14:58,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:14:58,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:14:59,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:00,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:01,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:02,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:03,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:03,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:04,111 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:15:04,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:05,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:06,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:07,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:08,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:09,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:09,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:10,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:11,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:12,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:12,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:15:12,840 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:15:13,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:14,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:15,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:15,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:16,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:17,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:18,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:19,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:20,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:20,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:21,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:22,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:23,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:24,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:25,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:25,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:15:25,633 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:15:25,634 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:15:25,636 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:15:26,092 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:15:26,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:26,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:27,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:27,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:15:27,840 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:15:28,094 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:15:28,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:29,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:30,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:31,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:31,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:32,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:33,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:34,114 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:15:34,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:35,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:36,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:37,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:37,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:38,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:39,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:40,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:41,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:42,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:42,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:15:42,840 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:15:42,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:43,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:44,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:45,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:46,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:47,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:48,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:48,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:49,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:50,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:51,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:52,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:53,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:53,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:54,423 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:55,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:56,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:57,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:57,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:15:57,653 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:15:57,654 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:15:57,656 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:15:58,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:15:58,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:15:58,129 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:15:58,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:15:59,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:15:59,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:00,130 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:16:00,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:01,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:02,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:03,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:04,118 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:16:04,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:04,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:05,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:06,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:07,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:08,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:09,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:09,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:10,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:11,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:12,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:13,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:16:13,070 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:16:13,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:14,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:14,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:15,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:16,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:17,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:18,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:19,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:20,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:20,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:21,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:22,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:23,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:24,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:25,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:25,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:26,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:27,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:28,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:16:28,076 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:16:28,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:29,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:29,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:16:29,578 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:16:29,579 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:16:29,580 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:16:30,164 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:16:30,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:30,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:31,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:32,165 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:16:32,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:33,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:34,122 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:16:34,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:35,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:36,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:36,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:37,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:38,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:39,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:40,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:41,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:41,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:42,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:43,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:16:43,076 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:16:43,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:44,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:45,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:46,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:46,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:47,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:48,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:49,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:50,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:51,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:52,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:52,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:53,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:54,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:55,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:56,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:57,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:58,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:16:58,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:16:58,081 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:16:58,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:16:59,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:16:59,309 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:16:59,309 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:16:59,310 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:16:59,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:00,200 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:17:00,201 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:17:00,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:01,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:02,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:03,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:03,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:04,126 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:17:04,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:05,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:06,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:07,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:08,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:09,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:09,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:10,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:11,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:12,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:13,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:17:13,080 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:17:13,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:14,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:14,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:15,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:16,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:17,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:18,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:19,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:19,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:20,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:21,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:22,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:23,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:24,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:25,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:25,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:26,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:27,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:28,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:17:28,080 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:17:28,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:28,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:17:28,643 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:17:28,643 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:17:28,645 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:17:29,229 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:17:29,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:30,230 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:17:30,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:31,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:31,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:32,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:33,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:34,131 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:17:34,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:35,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:36,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:37,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:37,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:38,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:39,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:40,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:41,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:42,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:43,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:43,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:17:43,080 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:17:43,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:44,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:45,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:46,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:47,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:48,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:48,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:49,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:50,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:51,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:52,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:53,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:54,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:54,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:55,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:56,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:57,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:58,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:17:58,089 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:17:58,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:17:59,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:17:59,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:00,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:01,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:02,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:02,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:18:02,901 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:18:02,901 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:18:02,902 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:18:03,265 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:18:03,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:04,134 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:18:04,266 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:18:04,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:05,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:05,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:06,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:07,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:08,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:09,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:10,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:11,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:11,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:12,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:13,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:18:13,083 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:18:13,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:14,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:15,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:16,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:16,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:17,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:18,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:19,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:20,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:21,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:22,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:22,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:23,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:24,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:25,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:26,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:27,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:27,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:28,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:18:28,091 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:18:28,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:29,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:29,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:18:29,601 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:18:29,602 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:18:29,603 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:18:30,286 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:18:30,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:31,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:32,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:32,287 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:18:32,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:33,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:34,137 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:18:34,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:35,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:36,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:37,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:38,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:38,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:39,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:40,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:41,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:42,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:43,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:18:43,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:18:43,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:43,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:44,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:45,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:46,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:47,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:48,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:49,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:49,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:50,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:51,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:52,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:53,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:54,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:54,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:55,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:56,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:57,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:58,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:18:58,089 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:18:58,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:18:58,296 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:18:58,297 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:18:58,298 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:18:58,316 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:18:58,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:18:59,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:18:59,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:00,318 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:19:00,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:01,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:02,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:03,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:04,141 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:19:04,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:05,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:05,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:06,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:07,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:08,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:09,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:10,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:11,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:11,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:12,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:13,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:19:13,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:19:13,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:14,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:15,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:16,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:16,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:17,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:18,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:19,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:20,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:21,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:22,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:22,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:23,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:24,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:25,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:26,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:27,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:28,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:28,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:19:28,148 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:19:28,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:29,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:30,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:31,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:31,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:19:31,907 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:19:31,908 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:19:31,909 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:19:32,349 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:19:32,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:33,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:33,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:34,151 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:19:34,351 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:19:34,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:35,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:36,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:37,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:38,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:39,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:39,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:40,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:41,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:42,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:43,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:19:43,144 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:19:43,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:44,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:44,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:45,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:46,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:47,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:48,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:49,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:50,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:50,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:51,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:52,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:53,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:54,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:55,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:56,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:19:56,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:57,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:58,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:19:58,144 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:19:58,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:19:59,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:00,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:01,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:01,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:02,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:03,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:04,148 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:20:04,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:20:04,403 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:20:04,403 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:20:04,404 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:20:04,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:05,378 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:20:05,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:06,379 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:20:06,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:07,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:07,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:08,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:09,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:10,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:11,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:12,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:13,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:13,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:20:13,144 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:20:13,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:14,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:15,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:16,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:17,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:18,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:18,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:19,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:20,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:21,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:22,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:23,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:23,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:24,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:25,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:26,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:27,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:28,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:28,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:20:28,144 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:20:28,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:29,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:30,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:31,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:32,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:33,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:33,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:34,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:20:34,008 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:20:34,008 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:20:34,009 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:20:34,154 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:20:34,405 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:20:34,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:35,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:36,407 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:20:36,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:37,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:38,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:39,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:39,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:40,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:41,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:42,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:43,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:20:43,144 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:20:43,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:44,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:44,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:45,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:46,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:47,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:48,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:49,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:50,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:50,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:51,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:52,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:53,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:54,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:55,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:20:55,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:56,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:57,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:58,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:20:58,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:20:58,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:20:59,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:00,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:00,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:01,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:02,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:03,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:04,155 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:21:04,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:21:04,352 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:21:04,352 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:21:04,354 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:21:04,436 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:21:04,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:05,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:06,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:06,438 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:21:06,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:07,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:08,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:09,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:10,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:11,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:11,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:12,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:13,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:21:13,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:21:13,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:14,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:15,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:16,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:17,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:17,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:18,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:19,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:20,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:21,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:22,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:22,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:23,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:24,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:25,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:26,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:27,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:28,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:28,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:21:28,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:21:28,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:29,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:30,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:31,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:32,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:33,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:33,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:34,159 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:21:34,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:21:34,730 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:21:34,730 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:21:34,731 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:21:35,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:35,468 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:21:36,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:36,469 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:21:37,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:38,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:38,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:39,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:40,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:41,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:42,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:43,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:43,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:21:43,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:21:43,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:44,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:45,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:46,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:47,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:48,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:49,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:49,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:50,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:51,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:52,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:53,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:54,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:55,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:21:55,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:56,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:57,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:58,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:21:58,149 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:21:58,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:21:59,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:00,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:00,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:01,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:02,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:02,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:22:02,939 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:22:02,939 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:22:02,941 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:22:03,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:03,499 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:22:04,163 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:22:04,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:04,500 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:22:05,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:06,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:06,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:07,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:08,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:09,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:10,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:11,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:11,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:12,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:13,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:22:13,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:22:13,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:14,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:15,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:16,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:16,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:17,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:18,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:19,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:20,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:21,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:22,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:22,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:23,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:24,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:25,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:26,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:27,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:27,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:28,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:22:28,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:22:28,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:29,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:30,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:30,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:22:30,370 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:22:30,371 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:22:30,372 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:22:30,524 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:22:31,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:32,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:32,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:32,526 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:22:33,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:34,168 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:22:34,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:35,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:36,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:37,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:38,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:38,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:39,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:40,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:41,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:42,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:43,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:22:43,149 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:22:43,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:43,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:44,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:45,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:46,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:47,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:48,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:49,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:49,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:50,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:51,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:52,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:53,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:54,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:55,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:22:55,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:56,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:57,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:58,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:22:58,157 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:22:58,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:22:59,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:00,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:00,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:01,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:01,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:23:01,783 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:23:01,783 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:23:01,785 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:23:02,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:02,551 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:23:02,552 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:23:03,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:04,172 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:23:04,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:05,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:06,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:06,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:07,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:08,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:09,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:10,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:11,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:12,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:12,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:13,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:23:13,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:23:13,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:14,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:15,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:16,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:17,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:17,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:18,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:19,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:20,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:21,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:22,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:22,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:23,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:24,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:25,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:26,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:27,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:28,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:23:28,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:23:28,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:28,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:29,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:30,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:31,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:32,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:33,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:34,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:34,174 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:23:34,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:34,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:23:34,751 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:23:34,751 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:23:34,753 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:23:35,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:35,580 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:23:36,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:36,581 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:23:37,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:38,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:39,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:40,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:40,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:41,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:42,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:43,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:23:43,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:23:43,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:44,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:45,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:46,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:46,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:47,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:48,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:49,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:50,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:51,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:51,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:52,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:53,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:54,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:55,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:56,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:57,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:23:57,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:58,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:23:58,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:23:58,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:23:59,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:00,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:01,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:02,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:02,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:03,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:04,178 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:24:04,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:05,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:05,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:24:05,764 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:24:05,764 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:24:05,766 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:24:06,605 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:24:06,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:07,606 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:24:07,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:07,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:08,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:09,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:10,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:11,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:12,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:13,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:24:13,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:24:13,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:13,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:14,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:15,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:16,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:17,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:18,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:19,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:19,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:20,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:21,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:22,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:23,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:24,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:25,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:25,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:26,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:27,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:28,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:24:28,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:24:28,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:29,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:30,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:31,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:31,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:32,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:33,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:34,182 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:24:34,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:35,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:36,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:36,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:37,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:37,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:24:37,985 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:24:37,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:24:37,987 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:24:38,638 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:24:38,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:39,639 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:24:39,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:40,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:41,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:42,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:42,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:43,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:24:43,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:24:43,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:44,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:45,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:46,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:47,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:48,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:48,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:49,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:50,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:51,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:52,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:53,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:53,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:54,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:55,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:56,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:57,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:58,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:24:58,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:24:58,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:24:58,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:24:59,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:00,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:01,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:02,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:03,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:04,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:04,188 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:25:04,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:05,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:06,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:07,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:08,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:09,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:09,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:09,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:25:09,864 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:25:09,864 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:25:09,866 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:25:10,666 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:25:10,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:11,667 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:25:11,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:12,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:13,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:25:13,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:25:13,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:14,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:14,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:15,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:16,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:17,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:18,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:19,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:19,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:20,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:21,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:22,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:23,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:24,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:25,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:25,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:26,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:27,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:28,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:25:28,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:25:28,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:29,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:30,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:31,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:31,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:32,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:33,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:34,192 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:25:34,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:35,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:36,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:37,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:37,865 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:38,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:39,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:40,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:41,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:42,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:42,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:43,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:25:43,160 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:25:43,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:25:43,850 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:25:43,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:25:43,851 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:25:44,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:44,700 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:25:44,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:45,701 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:25:45,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:46,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:47,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:48,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:48,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:49,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:51,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:52,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:53,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:54,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:54,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:25:55,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:56,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:57,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:58,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:25:58,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:25:58,158 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:25:59,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:00,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:00,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:01,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:02,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:03,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:04,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:04,197 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:26:05,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:05,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:06,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:07,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:08,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:09,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:10,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:10,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:11,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:12,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:13,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:13,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:26:13,162 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:26:14,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:15,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:15,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:26:15,142 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:26:15,142 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:26:15,144 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:26:15,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:15,723 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:26:16,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:17,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:17,725 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:26:18,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:19,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:20,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:21,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:21,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:22,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:23,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:24,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:25,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:26,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:27,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:27,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:28,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:28,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:26:28,159 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:26:29,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:30,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:31,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:32,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:32,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:33,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:34,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:34,201 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:26:35,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:36,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:37,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:38,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:38,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:39,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:40,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:41,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:42,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:43,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:43,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:26:43,163 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:26:44,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:44,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:26:44,194 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:26:44,195 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:26:44,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:44,196 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:26:44,761 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:26:45,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:45,762 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:26:46,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:47,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:48,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:49,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:49,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:50,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:51,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:52,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:53,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:54,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:55,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:55,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:26:56,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:57,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:58,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:26:58,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:26:58,264 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:26:59,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:00,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:00,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:01,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:02,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:03,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:04,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:04,205 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:27:05,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:06,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:06,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:07,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:08,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:09,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:10,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:11,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:11,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:12,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:13,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:13,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:27:13,165 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:27:14,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:15,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:15,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:27:15,434 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:27:15,434 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:27:15,436 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:27:15,799 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:27:16,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:16,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:17,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:17,801 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:27:18,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:19,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:20,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:21,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:22,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:22,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:23,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:24,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:25,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:26,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:27,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:28,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:27:28,232 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:27:28,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:28,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:29,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:30,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:31,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:32,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:33,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:33,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:34,209 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:27:34,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:35,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:36,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:37,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:38,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:39,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:39,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:40,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:41,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:42,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:43,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:27:43,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:27:43,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:44,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:45,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:45,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:46,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:46,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:27:46,555 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:27:46,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:27:46,556 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:27:46,827 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:27:47,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:47,828 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:27:48,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:49,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:50,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:50,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:51,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:52,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:53,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:54,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:55,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:55,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:27:56,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:57,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:58,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:27:58,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:27:58,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:27:59,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:00,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:00,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:01,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:02,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:03,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:04,214 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:28:04,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:05,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:05,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:06,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:07,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:08,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:09,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:10,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:11,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:11,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:12,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:13,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:28:13,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:28:13,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:14,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:15,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:16,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:16,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:17,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:17,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:28:17,947 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:28:17,948 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:28:17,949 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:28:18,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:18,851 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:28:19,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:19,852 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:28:20,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:21,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:22,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:22,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:23,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:24,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:25,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:26,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:27,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:28,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:28:28,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:28:28,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:28,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:29,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:30,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:31,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:32,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:33,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:33,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:34,218 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:28:34,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:35,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:36,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:37,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:38,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:39,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:39,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:40,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:41,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:42,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:43,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:28:43,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:28:43,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:44,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:45,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:45,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:46,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:47,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:48,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:49,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:50,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:50,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:51,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:28:51,237 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:28:51,237 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:28:51,238 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:28:51,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:51,876 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:28:52,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:53,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:53,878 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:28:54,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:55,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:28:55,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:56,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:57,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:58,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:28:58,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:28:58,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:28:59,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:00,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:01,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:01,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:02,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:03,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:04,222 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:29:04,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:05,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:06,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:07,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:07,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:08,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:09,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:10,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:11,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:12,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:13,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:29:13,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:29:13,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:13,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:14,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:15,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:16,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:17,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:18,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:18,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:19,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:20,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:21,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:29:21,317 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:29:21,318 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:29:21,319 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:29:21,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:21,912 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:29:22,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:23,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:23,914 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:29:24,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:24,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:25,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:26,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:27,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:28,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:29:28,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:29:28,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:29,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:29,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:30,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:31,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:32,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:33,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:34,226 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:29:34,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:35,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:35,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:36,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:37,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:38,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:39,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:40,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:40,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:41,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:42,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:43,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:29:43,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:29:43,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:44,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:45,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:45,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:46,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:47,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:48,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:49,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:50,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:50,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:51,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:52,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:53,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:53,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:29:53,829 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:29:53,830 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:29:53,831 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:29:53,942 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:29:54,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:55,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:29:55,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:55,943 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:29:56,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:57,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:58,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:29:58,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:29:58,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:29:59,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:00,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:01,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:01,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:02,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:03,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:04,230 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:30:04,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:05,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:06,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:07,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:07,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:08,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:09,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:10,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:11,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:12,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:12,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:13,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:30:13,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:30:13,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:14,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:15,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:16,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:17,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:18,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:18,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:19,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:20,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:21,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:22,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:23,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:23,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:24,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:30:24,624 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:30:24,624 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:30:24,625 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:30:24,968 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:30:25,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:25,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:25,969 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:30:26,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:27,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:28,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:30:28,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:30:28,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:29,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:29,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:30,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:31,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:32,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:33,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:34,233 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:30:34,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:35,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:35,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:36,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:37,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:38,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:39,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:40,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:40,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:41,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:42,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:43,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:30:43,234 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:30:43,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:44,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:45,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:45,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:46,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:47,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:48,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:49,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:50,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:50,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:51,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:52,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:53,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:54,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:55,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:30:55,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:56,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:30:56,282 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:30:56,283 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:30:56,285 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:30:56,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:56,998 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:30:57,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:58,000 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:30:58,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:30:58,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:30:58,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:30:59,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:00,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:00,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:01,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:02,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:03,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:04,237 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:31:04,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:05,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:06,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:06,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:07,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:08,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:09,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:10,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:11,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:12,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:12,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:13,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:31:13,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:31:13,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:14,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:15,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:16,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:17,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:18,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:18,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:19,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:20,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:21,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:22,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:23,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:23,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:24,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:25,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:26,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:26,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:31:26,994 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:31:26,995 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:31:26,996 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:31:27,033 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:31:27,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:28,034 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:31:28,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:31:28,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:31:28,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:29,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:29,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:30,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:31,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:32,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:33,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:34,241 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:31:34,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:35,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:35,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:36,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:37,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:38,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:39,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:40,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:40,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:41,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:42,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:43,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:31:43,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:31:43,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:44,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:45,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:46,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:46,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:47,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:48,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:49,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:50,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:51,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:51,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:52,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:53,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:54,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:55,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:31:55,221 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:31:55,221 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:31:55,222 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:31:55,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:56,062 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:31:56,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:31:56,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:57,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:58,063 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:31:58,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:31:58,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:31:58,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:31:59,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:00,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:01,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:02,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:02,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:03,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:04,245 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:32:04,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:05,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:06,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:07,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:08,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:08,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:09,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:10,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:11,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:12,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:13,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:32:13,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:32:13,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:14,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:14,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:15,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:16,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:17,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:18,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:19,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:19,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:20,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:21,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:22,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:23,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:24,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:24,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:25,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:26,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:27,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:32:27,736 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:32:27,736 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:32:27,737 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:32:28,087 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:32:28,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:28,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:32:28,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:32:29,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:30,090 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:32:30,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:30,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:31,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:32,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:33,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:34,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:34,249 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:32:35,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:35,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:36,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:37,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:38,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:39,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:40,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:40,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:41,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:42,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:43,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:43,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:32:43,235 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:32:44,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:45,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:46,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:46,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:47,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:48,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:49,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:50,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:51,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:52,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:52,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:53,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:54,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:55,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:56,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:57,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:57,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:32:58,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:32:58,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:32:58,242 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:32:59,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:00,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:01,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:01,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:33:01,414 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:33:01,414 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:33:01,416 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:33:02,114 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:33:02,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:03,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:03,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:04,115 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:33:04,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:04,253 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:33:05,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:06,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:07,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:08,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:08,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:09,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:10,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:11,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:12,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:13,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:13,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:33:13,241 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:33:14,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:14,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:15,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:16,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:17,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:18,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:19,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:19,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:20,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:21,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:22,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:23,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:24,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:24,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:25,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:26,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:27,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:28,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:28,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:33:28,254 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:33:29,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:30,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:30,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:31,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:31,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:33:31,573 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:33:31,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:33:31,576 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:33:32,141 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:33:32,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:33,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:34,142 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:33:34,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:34,257 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:33:35,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:35,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:36,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:37,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:38,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:39,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:40,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:41,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:41,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:42,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:43,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:43,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:33:43,248 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:33:44,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:45,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:46,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:46,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:47,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:48,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:49,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:50,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:51,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:52,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:52,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:53,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:54,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:55,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:56,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:57,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:57,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:33:58,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:33:58,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:33:58,257 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:33:59,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:00,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:01,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:01,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:34:01,308 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:34:01,308 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:34:01,310 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:34:02,169 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:34:02,170 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:34:02,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:03,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:03,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:04,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:04,262 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:34:05,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:06,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:07,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:08,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:08,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:09,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:10,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:11,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:12,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:13,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:13,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:34:13,254 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:34:14,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:14,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:15,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:16,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:17,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:18,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:19,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:19,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:20,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:21,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:22,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:23,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:24,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:25,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:25,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:26,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:27,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:28,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:28,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:34:28,254 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:34:29,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:30,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:30,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:31,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:32,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:32,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:34:32,528 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:34:32,528 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:34:32,530 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:34:33,197 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:34:33,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:34,198 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:34:34,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:34,266 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:34:35,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:35,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:36,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:37,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:38,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:39,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:40,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:40,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:41,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:42,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:43,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:43,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:34:43,254 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:34:44,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:45,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:45,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:46,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:47,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:48,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:49,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:50,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:51,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:51,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:52,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:53,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:54,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:55,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:56,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:56,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:34:57,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:58,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:34:58,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:34:58,264 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:34:59,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:00,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:01,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:01,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:02,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:35:02,084 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:35:02,084 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:35:02,086 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:35:02,229 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:35:02,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:03,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:04,230 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:35:04,270 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:35:04,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:05,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:06,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:07,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:07,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:08,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:09,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:10,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:11,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:12,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:13,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:35:13,261 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:35:13,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:13,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:14,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:15,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:16,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:17,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:18,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:18,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:19,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:20,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:21,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:22,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:23,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:23,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:24,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:25,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:26,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:27,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:28,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:35:28,271 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:35:28,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:29,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:29,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:29,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:35:29,932 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:35:29,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:35:29,941 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:35:30,253 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:35:30,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:31,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:32,255 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:35:32,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:33,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:34,274 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:35:34,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:35,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:35,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:36,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:37,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:38,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:39,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:40,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:40,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:41,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:42,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:43,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:35:43,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:35:43,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:44,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:45,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:45,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:46,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:47,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:48,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:49,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:50,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:50,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:51,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:52,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:53,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:54,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:55,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:55,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:35:56,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:57,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:58,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:35:58,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:35:58,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:35:59,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:00,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:00,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:01,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:01,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:36:01,652 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:36:01,653 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:36:01,654 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:36:02,282 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:36:02,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:03,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:04,278 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:36:04,285 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:36:04,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:05,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:06,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:06,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:07,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:08,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:09,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:10,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:11,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:12,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:12,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:13,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:36:13,268 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:36:13,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:14,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:15,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:16,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:17,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:17,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:18,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:19,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:20,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:21,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:22,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:23,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:23,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:24,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:25,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:26,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:27,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:28,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:36:28,271 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:36:28,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:29,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:29,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:30,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:31,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:31,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:36:31,988 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:36:31,988 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:36:31,990 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:36:32,316 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:36:32,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:33,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:34,283 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:36:34,318 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:36:34,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:35,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:35,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:36,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:37,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:38,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:39,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:40,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:40,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:41,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:42,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:43,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:36:43,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:36:43,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:44,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:45,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:45,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:46,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:47,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:48,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:49,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:50,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:50,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:51,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:52,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:53,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:54,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:36:55,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:36:56,331 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:36:58,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:00,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:04,286 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:37:04,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:05,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:37:05,619 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:37:06,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:11,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:12,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:13,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:14,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:17,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:21,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:37:21,107 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:37:21,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:22,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:22,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:27,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:29,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:31,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:32,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:34,290 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:37:38,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:39,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:37:39,785 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:37:40,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:43,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:47,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:48,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:51,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:52,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:53,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:37:53,007 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:37:53,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:54,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:54,806 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:37:54,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:55,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:56,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:57,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:58,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:37:59,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:37:59,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:00,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:01,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:02,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:03,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:04,294 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:38:04,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:05,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:05,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:06,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:07,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:08,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:38:08,007 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:38:08,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:09,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:10,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:11,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:11,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:12,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:13,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:14,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:15,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:16,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:16,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:17,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:18,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:19,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:20,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:21,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:22,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:22,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:23,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:38:23,007 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:38:23,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:24,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:25,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:26,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:27,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:27,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:28,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:29,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:30,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:31,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:32,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:33,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:33,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:34,299 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:38:34,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:35,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:36,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:37,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:38,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:38:38,008 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:38:38,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:39,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:39,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:40,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:41,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:42,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:43,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:38:44,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:49,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:54,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:38:59,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:04,303 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:39:04,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:39:04,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:39:04,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:04,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:05,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:06,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:07,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:08,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:09,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:10,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:10,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:11,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:12,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:13,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:14,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:15,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:15,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:16,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:17,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:18,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:19,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:39:19,761 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:39:19,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:20,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:20,865 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:21,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:22,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:23,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:24,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:25,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:26,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:26,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:27,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:28,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:29,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:30,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:31,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:31,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:32,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:33,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:34,308 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:39:34,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:39:34,761 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:39:34,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:35,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:36,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:36,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:37,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:38,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:39,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:40,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:41,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:42,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:42,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:43,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:44,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:45,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:46,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:47,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:47,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:48,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:49,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:39:49,761 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:39:49,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:50,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:51,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:52,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:52,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:53,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:54,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:55,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:56,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:57,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:58,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:39:58,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:39:59,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:00,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:01,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:02,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:03,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:03,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:04,311 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:40:04,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:40:04,761 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:40:04,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:05,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:06,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:07,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:08,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:08,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:09,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:10,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:11,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:12,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:13,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:14,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:14,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:15,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:16,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:17,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:18,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:19,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:40:19,761 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:40:19,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:19,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:20,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:21,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:22,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:23,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:24,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:25,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:25,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:26,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:27,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:28,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:29,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:30,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:30,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:31,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:32,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:33,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:34,315 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:40:34,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:40:34,762 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:40:34,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:35,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:35,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:36,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:37,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:38,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:39,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:40,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:41,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:41,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:42,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:43,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:44,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:45,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:46,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:47,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:47,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:40:52,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:40:57,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:02,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:04,320 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:41:08,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:13,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:18,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:23,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:27,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:41:27,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:27,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:41:28,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:28,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:29,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:30,268 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:41:30,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:31,269 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:41:31,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:32,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:33,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:34,272 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:41:34,321 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:41:34,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:34,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:35,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:36,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:37,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:38,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:39,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:39,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:40,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:41,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:42,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:41:42,363 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:41:42,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:43,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:44,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:44,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:45,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:46,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:47,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:48,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:49,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:50,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:50,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:51,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:52,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:53,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:54,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:55,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:41:55,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:56,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:57,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:41:57,370 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:41:57,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:58,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:41:59,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:00,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:42:00,040 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:42:00,040 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:42:00,043 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:42:00,299 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:42:00,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:01,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:01,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:02,301 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:42:02,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:03,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:04,324 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:42:04,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:05,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:06,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:06,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:07,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:08,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:09,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:10,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:11,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:11,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:12,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:42:12,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:42:12,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:13,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:14,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:15,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:16,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:16,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:17,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:18,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:19,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:20,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:21,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:22,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:22,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:23,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:24,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:25,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:26,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:27,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:42:27,371 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:42:27,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:27,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:28,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:29,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:29,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:42:29,819 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:42:29,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:42:29,820 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:42:30,324 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:42:30,325 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:42:30,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:31,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:32,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:33,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:33,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:34,328 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:42:34,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:35,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:36,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:37,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:38,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:39,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:39,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:40,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:41,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:42,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:42:42,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:42:42,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:43,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:44,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:44,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:45,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:46,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:47,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:48,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:49,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:50,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:50,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:51,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:52,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:53,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:54,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:55,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:42:55,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:56,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:42:56,580 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:42:56,581 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:42:56,582 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:42:57,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:57,351 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:42:57,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:42:57,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:42:58,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:59,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:42:59,352 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:43:00,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:00,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:01,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:02,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:03,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:04,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:04,332 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:43:05,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:06,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:06,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:07,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:08,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:09,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:10,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:11,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:11,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:12,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:12,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:43:12,368 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:43:13,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:14,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:15,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:16,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:16,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:17,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:18,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:19,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:20,014 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:21,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:22,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:22,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:23,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:24,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:25,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:25,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:43:25,696 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:43:25,697 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:43:25,699 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:43:26,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:26,379 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:43:27,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:27,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:43:27,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:43:27,380 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:43:27,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:28,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:29,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:30,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:31,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:32,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:33,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:33,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:34,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:34,336 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:43:35,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:36,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:37,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:38,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:39,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:39,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:40,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:41,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:42,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:42,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:43:42,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:43:43,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:44,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:44,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:45,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:46,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:47,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:48,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:49,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:50,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:50,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:51,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:52,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:53,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:54,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:55,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:55,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:43:56,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:57,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:57,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:43:57,372 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:43:58,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:58,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:43:58,468 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:43:58,468 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:43:58,470 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:43:59,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:43:59,408 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:44:00,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:00,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:01,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:01,410 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:44:02,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:03,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:04,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:04,340 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:44:05,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:06,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:06,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:07,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:08,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:09,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:10,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:11,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:11,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:12,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:12,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:44:12,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:44:13,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:14,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:15,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:16,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:16,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:17,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:18,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:19,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:20,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:21,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:22,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:22,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:23,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:24,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:25,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:26,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:26,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:44:26,755 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:44:26,755 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:44:26,757 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:44:27,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:27,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:44:27,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:44:27,433 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:44:27,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:28,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:29,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:29,435 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:44:30,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:31,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:32,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:33,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:33,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:34,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:34,344 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:44:35,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:36,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:37,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:38,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:39,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:39,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:40,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:41,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:42,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:42,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:44:42,369 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:44:43,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:44,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:44,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:45,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:46,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:47,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:48,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:49,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:50,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:50,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:51,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:52,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:53,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:54,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:55,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:55,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:44:56,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:57,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:57,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:44:57,380 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:44:58,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:44:59,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:00,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:00,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:00,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:45:00,851 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:45:00,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:45:00,852 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:45:01,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:01,465 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:45:02,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:03,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:03,466 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:45:04,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:04,349 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:45:05,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:06,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:06,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:07,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:08,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:09,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:10,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:11,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:11,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:12,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:12,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:45:12,376 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:45:13,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:14,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:15,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:16,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:16,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:17,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:18,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:19,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:20,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:21,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:22,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:22,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:23,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:24,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:25,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:26,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:27,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:27,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:45:27,382 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:45:27,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:28,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:29,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:45:29,285 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:45:29,286 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:45:29,287 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:45:29,490 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:45:29,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:30,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:31,492 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:45:31,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:32,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:33,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:33,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:34,352 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:45:34,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:35,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:36,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:37,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:38,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:38,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:39,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:40,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:41,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:42,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:45:42,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:45:42,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:43,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:43,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:44,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:45,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:46,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:47,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:48,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:49,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:49,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:50,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:51,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:52,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:53,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:54,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:45:54,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:55,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:56,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:57,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:45:57,520 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:45:57,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:58,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:45:59,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:45:59,156 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:45:59,157 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:45:59,158 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:45:59,529 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:45:59,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:00,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:00,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:01,531 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:46:01,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:02,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:03,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:04,356 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:46:04,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:05,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:05,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:06,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:07,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:08,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:09,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:10,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:10,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:11,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:12,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:46:12,518 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:46:12,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:13,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:14,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:15,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:15,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:16,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:17,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:18,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:19,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:20,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:21,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:21,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:22,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:23,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:24,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:25,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:26,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:26,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:27,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:46:27,527 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:46:27,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:28,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:29,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:30,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:30,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:46:30,750 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:46:30,750 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:46:30,752 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:46:31,557 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:46:31,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:31,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:32,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:33,559 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:46:33,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:34,360 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:46:34,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:35,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:36,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:37,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:37,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:38,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:39,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:40,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:41,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:42,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:46:42,523 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:46:42,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:42,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:43,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:44,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:45,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:46,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:47,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:48,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:48,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:49,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:50,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:51,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:52,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:53,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:54,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:54,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:55,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:56,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:57,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:46:57,523 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:46:57,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:58,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:46:59,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:46:59,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:00,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:01,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:02,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:03,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:47:03,696 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:47:03,696 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:47:03,697 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:47:04,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:04,365 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:47:04,584 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:47:05,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:05,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:05,586 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:47:06,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:07,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:08,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:09,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:10,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:10,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:11,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:12,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:12,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:47:12,523 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:47:13,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:14,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:15,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:15,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:16,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:17,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:18,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:19,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:20,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:21,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:21,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:22,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:23,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:24,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:25,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:26,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:27,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:27,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:47:27,523 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:47:27,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:28,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:29,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:30,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:31,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:32,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:33,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:33,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:34,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:34,370 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:47:34,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:47:34,915 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:47:34,915 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:47:34,916 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:47:35,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:35,616 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:47:36,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:37,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:37,618 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:47:38,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:39,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:39,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:40,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:41,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:42,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:42,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:47:42,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:47:43,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:44,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:44,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:45,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:46,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:47,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:48,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:49,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:50,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:50,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:51,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:52,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:53,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:54,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:55,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:56,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:56,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:47:57,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:57,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:47:57,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:47:58,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:47:59,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:00,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:01,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:01,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:02,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:03,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:04,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:04,376 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:48:05,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:06,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:06,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:48:06,461 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:48:06,462 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:48:06,463 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:48:06,644 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:48:07,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:07,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:08,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:09,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:09,646 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:48:10,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:11,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:12,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:12,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:48:12,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:48:12,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:13,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:14,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:15,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:16,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:17,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:18,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:18,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:19,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:20,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:21,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:22,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:23,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:23,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:24,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:25,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:26,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:27,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:27,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:48:27,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:48:28,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:28,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:29,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:30,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:31,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:32,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:33,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:34,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:34,384 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:48:34,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:35,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:35,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:48:35,591 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:48:35,591 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:48:35,593 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:48:35,668 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:48:36,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:37,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:37,670 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:48:38,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:39,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:39,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:40,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:41,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:42,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:42,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:48:42,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:48:43,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:44,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:44,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:45,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:46,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:47,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:48,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:49,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:50,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:50,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:51,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:52,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:53,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:54,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:55,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:55,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:48:56,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:57,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:57,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:48:57,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:48:58,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:48:59,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:00,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:00,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:01,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:02,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:03,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:04,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:04,382 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:49:05,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:05,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:49:05,417 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:49:05,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:49:05,418 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:49:05,695 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:49:06,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:06,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:07,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:07,696 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:49:08,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:09,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:10,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:11,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:11,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:12,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:12,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:49:12,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:49:13,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:14,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:15,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:16,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:16,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:17,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:18,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:19,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:20,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:21,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:22,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:22,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:23,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:24,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:25,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:26,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:27,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:27,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:49:27,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:49:27,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:28,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:29,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:30,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:31,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:32,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:33,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:33,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:34,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:34,386 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:49:34,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:49:34,467 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:49:34,467 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:49:34,469 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:49:34,723 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:49:35,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:35,725 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:49:36,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:37,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:38,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:39,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:39,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:40,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:41,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:42,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:42,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:49:42,525 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:49:43,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:44,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:44,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:45,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:46,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:47,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:48,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:49,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:49,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:50,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:51,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:52,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:53,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:54,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:55,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:55,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:49:56,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:57,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:57,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:49:57,525 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:49:58,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:49:59,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:00,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:00,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:01,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:02,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:03,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:04,388 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:50:04,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:05,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:50:05,201 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:50:05,201 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:50:05,203 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:50:05,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:05,756 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:50:06,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:06,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:07,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:07,757 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:50:08,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:09,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:10,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:11,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:11,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:12,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:50:12,525 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:50:12,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:13,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:14,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:15,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:16,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:17,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:17,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:18,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:19,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:20,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:21,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:22,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:22,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:23,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:24,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:25,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:26,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:27,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:50:27,525 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:50:27,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:27,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:28,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:29,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:30,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:31,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:32,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:33,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:33,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:34,400 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:50:34,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:35,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:36,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:37,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:38,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:38,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:39,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:40,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:50:40,365 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:50:40,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:50:40,367 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:50:40,782 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:50:41,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:41,784 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:50:42,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:42,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:50:42,526 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:50:43,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:44,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:44,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:45,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:46,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:47,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:48,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:49,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:50,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:50,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:51,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:52,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:53,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:54,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:55,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:56,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:56,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:50:57,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:57,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:50:57,526 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:50:58,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:50:59,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:00,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:01,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:01,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:02,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:03,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:04,396 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:51:04,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:05,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:06,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:07,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:07,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:08,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:09,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:10,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:11,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:12,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:12,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:51:12,530 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:51:12,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:13,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:51:13,075 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:51:13,076 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:51:13,079 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:51:13,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:13,812 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:51:14,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:15,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:15,814 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:51:16,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:17,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:17,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:18,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:19,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:20,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:21,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:22,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:23,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:23,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:24,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:25,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:26,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:27,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:51:27,530 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:51:27,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:28,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:28,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:29,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:30,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:31,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:32,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:33,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:33,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:34,400 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:51:34,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:35,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:36,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:37,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:38,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:39,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:39,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:40,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:41,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:42,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:51:42,539 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:51:42,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:43,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:44,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:44,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:45,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:51:45,430 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:51:45,431 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:51:45,437 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:51:45,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:45,854 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:51:46,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:47,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:47,856 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:51:48,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:49,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:50,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:50,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:51,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:52,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:53,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:54,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:55,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:56,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:51:57,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:57,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:51:57,534 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:51:58,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:51:59,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:00,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:01,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:01,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:02,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:03,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:04,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:04,405 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:52:05,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:06,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:07,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:07,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:08,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:09,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:10,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:11,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:12,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:12,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:52:12,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:52:12,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:13,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:14,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:14,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:52:14,579 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:52:14,579 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:52:14,581 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:52:14,884 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:52:15,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:15,885 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:52:16,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:17,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:17,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:18,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:19,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:20,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:21,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:22,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:23,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:23,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:24,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:25,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:26,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:27,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:27,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:52:27,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:52:28,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:29,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:29,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:30,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:31,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:32,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:33,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:34,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:34,409 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:52:35,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:35,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:36,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:37,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:38,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:39,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:40,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:40,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:41,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:42,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:42,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:52:42,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:52:43,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:44,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:45,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:45,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:46,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:47,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:47,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:52:47,664 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:52:47,664 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:52:47,666 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:52:47,915 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:52:48,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:49,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:49,917 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:52:50,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:50,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:51,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:52,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:53,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:54,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:55,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:56,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:56,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:52:57,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:57,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:52:57,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:52:58,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:52:59,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:00,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:01,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:02,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:02,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:03,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:04,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:04,414 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:53:05,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:06,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:07,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:07,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:08,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:09,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:10,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:11,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:12,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:12,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:53:12,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:53:12,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:13,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:14,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:15,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:16,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:17,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:18,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:18,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:53:18,642 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:53:18,642 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:53:18,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:18,643 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:53:18,942 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:53:19,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:19,943 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:53:20,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:21,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:22,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:23,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:23,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:24,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:25,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:26,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:27,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:27,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:53:27,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:53:28,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:29,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:29,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:30,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:31,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:32,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:33,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:34,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:34,416 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:53:35,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:35,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:36,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:37,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:38,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:39,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:40,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:40,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:41,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:42,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:42,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:53:42,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:53:43,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:44,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:45,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:45,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:46,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:47,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:48,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:49,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:50,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:53:50,117 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:53:50,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:53:50,119 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:53:50,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:50,968 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:53:51,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:51,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:51,969 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:53:52,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:53,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:54,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:55,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:56,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:56,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:53:57,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:57,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:53:57,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:53:58,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:53:59,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:00,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:01,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:02,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:02,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:03,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:04,420 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:54:04,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:05,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:06,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:07,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:07,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:08,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:09,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:10,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:11,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:12,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:12,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:54:12,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:54:12,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:13,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:14,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:15,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:16,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:17,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:18,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:18,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:19,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:20,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:21,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:21,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:54:21,987 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:54:21,988 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:54:21,990 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:54:22,001 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:54:22,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:23,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:23,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:24,003 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:54:24,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:25,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:26,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:27,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:54:27,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:54:27,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:28,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:29,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:29,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:30,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:31,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:32,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:33,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:34,424 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:54:34,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:35,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:35,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:36,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:37,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:38,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:39,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:40,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:40,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:41,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:42,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:54:42,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:54:42,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:43,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:44,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:45,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:45,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:46,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:47,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:48,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:49,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:50,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:54:50,023 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:54:50,024 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:54:50,025 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:54:50,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:51,025 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:54:51,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:51,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:52,027 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:54:52,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:53,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:54,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:55,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:56,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:56,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:54:57,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:54:57,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:54:57,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:58,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:54:59,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:00,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:01,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:01,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:02,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:03,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:04,428 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:55:04,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:05,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:06,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:07,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:07,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:08,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:09,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:10,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:11,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:12,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:55:12,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:55:12,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:12,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:13,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:14,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:15,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:16,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:17,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:17,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:18,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:19,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:20,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:55:20,168 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:55:20,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:55:20,170 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:55:20,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:21,053 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:55:21,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:22,054 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:55:22,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:23,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:23,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:24,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:25,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:26,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:27,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:55:27,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:55:27,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:28,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:29,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:29,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:30,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:31,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:32,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:33,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:34,433 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:55:34,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:35,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:35,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:36,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:37,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:38,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:39,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:40,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:40,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:41,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:42,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:55:42,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:55:42,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:43,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:44,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:45,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:45,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:46,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:47,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:48,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:49,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:55:49,538 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:55:49,539 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:55:49,540 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:55:50,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:50,081 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:55:50,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:51,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:51,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:52,082 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:55:52,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:53,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:54,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:55,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:56,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:55:56,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:57,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:55:57,539 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:55:57,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:58,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:55:59,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:00,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:01,789 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:01,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:02,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:03,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:04,437 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:56:04,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:05,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:06,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:07,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:07,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:08,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:09,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:10,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:11,961 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:12,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:56:12,548 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:56:12,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:12,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:13,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:14,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:15,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:16,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:17,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:56:17,744 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:56:17,744 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:56:17,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:17,746 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:56:18,111 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:56:18,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:19,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:20,113 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:56:20,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:21,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:22,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:22,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:23,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:24,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:25,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:26,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:27,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:27,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:56:27,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:56:27,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:28,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:29,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:30,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:31,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:32,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:32,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:33,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:34,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:34,441 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:56:35,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:36,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:37,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:38,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:38,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:39,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:40,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:41,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:42,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:42,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:56:42,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:56:43,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:44,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:44,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:45,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:46,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:47,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:48,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:49,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:49,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:50,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:51,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:56:51,168 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:56:51,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:56:51,170 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:56:51,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:52,143 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:56:52,144 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:56:52,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:53,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:54,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:55,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:55,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:56:56,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:57,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:57,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:56:57,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:56:58,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:56:59,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:00,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:01,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:01,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:02,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:03,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:04,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:04,445 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:57:05,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:06,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:07,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:07,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:08,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:09,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:10,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:11,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:12,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:12,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:57:12,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:57:12,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:13,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:14,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:15,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:16,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:17,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:17,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:18,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:19,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:20,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:21,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:22,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:22,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:23,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:24,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:57:24,085 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:57:24,085 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:57:24,087 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:57:24,173 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:57:24,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:25,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:26,175 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:57:26,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:27,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:27,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:57:27,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:57:28,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:28,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:29,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:30,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:31,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:32,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:33,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:34,449 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:57:34,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:34,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:35,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:36,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:37,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:38,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:39,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:39,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:40,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:41,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:42,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:42,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:57:42,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:57:43,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:44,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:45,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:45,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:46,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:47,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:48,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:49,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:50,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:50,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:51,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:52,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:53,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:57:53,264 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:57:53,264 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:57:53,266 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:57:53,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:54,200 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:57:54,200 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:57:54,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:55,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:56,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:57:56,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:57,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:57:57,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:57:57,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:58,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:57:59,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:00,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:01,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:01,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:02,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:03,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:04,453 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:58:04,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:05,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:06,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:07,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:07,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:08,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:09,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:10,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:11,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:12,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:58:12,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:58:12,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:12,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:13,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:14,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:15,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:16,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:17,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:17,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:18,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:19,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:20,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:21,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:58:21,395 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:58:21,395 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:58:21,397 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:58:21,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:22,237 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:58:22,238 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:58:22,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:23,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:23,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:24,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:25,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:26,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:27,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:58:27,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:58:27,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:28,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:29,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:29,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:30,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:31,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:32,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:33,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:34,458 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:58:34,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:35,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:35,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:36,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:37,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:38,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:39,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:40,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:40,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:41,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:42,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:58:42,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:58:42,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:43,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:44,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:45,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:45,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:46,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:47,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:48,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:49,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:58:49,598 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:58:49,599 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:58:49,600 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:58:50,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:50,269 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:58:50,269 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:58:51,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:51,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:52,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:53,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:54,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:55,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:56,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:56,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:58:57,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:57,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:58:57,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:58:58,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:58:59,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:00,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:01,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:02,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:02,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:03,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:04,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:04,462 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:59:05,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:06,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:07,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:07,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:08,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:09,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:10,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:11,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:12,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:12,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:59:12,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:59:13,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:13,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:14,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:15,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:16,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:17,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:18,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:18,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:19,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:20,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:21,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:22,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:23,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:59:23,036 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:59:23,036 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:59:23,038 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:59:23,299 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:59:23,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:24,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:24,300 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:59:24,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:25,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:26,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:27,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:27,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:59:27,543 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:59:28,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:29,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:29,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:30,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:31,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:32,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:33,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:34,466 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 16:59:34,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:35,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:35,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:36,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:37,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:38,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:39,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:40,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:40,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:41,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:42,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:59:42,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:59:42,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:43,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:44,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:45,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:46,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:46,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:47,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:48,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:49,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:50,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:51,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:51,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 16:59:51,915 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 16:59:51,916 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 16:59:51,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:51,918 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 16:59:52,326 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 16:59:52,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:53,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:54,328 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 16:59:54,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:55,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:56,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:57,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 16:59:57,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 16:59:57,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:57,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 16:59:58,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 16:59:59,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:00,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:01,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:02,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:02,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:03,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:04,470 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:00:04,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:05,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:06,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:07,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:07,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:08,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:09,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:10,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:11,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:12,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:00:12,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:00:12,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:13,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:13,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:14,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:15,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:16,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:17,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:18,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:18,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:19,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:20,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:21,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:00:21,253 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:00:21,254 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:00:21,255 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:00:21,354 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:00:21,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:22,355 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:00:22,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:23,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:24,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:24,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:25,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:26,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:27,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:00:27,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:00:27,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:28,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:29,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:30,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:30,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:31,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:32,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:33,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:34,474 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:00:34,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:35,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:36,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:36,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:37,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:38,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:39,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:40,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:41,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:41,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:42,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:00:42,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:00:42,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:43,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:44,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:45,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:46,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:46,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:47,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:48,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:49,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:50,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:51,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:00:51,081 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:00:51,081 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:00:51,083 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:00:51,391 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:00:51,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:52,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:52,392 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:00:52,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:53,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:54,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:55,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:56,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:57,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:00:57,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:00:57,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:00:57,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:58,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:00:59,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:00,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:01,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:02,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:02,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:03,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:04,479 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:01:05,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:05,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:06,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:07,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:08,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:08,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:09,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:10,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:11,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:12,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:01:12,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:01:12,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:13,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:14,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:14,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:15,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:16,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:17,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:18,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:19,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:19,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:20,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:21,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:22,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:23,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:01:23,453 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:01:23,453 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:01:23,455 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:01:23,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:24,422 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:01:24,422 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:01:24,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:25,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:25,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:26,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:27,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:01:27,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:01:27,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:28,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:29,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:30,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:30,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:31,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:32,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:33,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:34,483 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:01:34,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:35,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:36,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:36,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:37,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:38,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:39,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:40,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:41,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:41,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:42,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:01:42,544 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:01:42,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:43,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:44,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:45,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:46,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:47,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:47,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:48,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:49,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:50,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:51,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:52,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:52,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:53,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:01:53,498 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:01:53,499 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:01:53,500 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:01:53,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:54,447 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:01:54,447 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:01:54,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:55,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:56,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:57,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:01:57,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:01:57,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:57,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:01:58,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:01:59,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:00,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:01,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:02,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:02,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:03,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:04,487 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:02:04,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:05,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:06,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:07,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:07,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:08,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:09,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:10,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:11,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:12,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:02:12,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:02:12,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:13,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:13,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:14,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:15,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:16,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:17,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:18,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:18,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:19,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:20,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:21,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:22,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:23,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:02:23,233 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:02:23,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:02:23,235 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:02:23,471 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:02:23,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:24,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:24,472 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:02:24,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:25,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:26,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:27,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:02:27,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:02:27,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:28,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:29,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:29,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:30,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:31,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:33,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:34,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:34,491 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:02:35,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:35,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:36,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:37,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:38,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:39,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:40,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:40,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:41,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:42,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:42,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:02:42,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:02:43,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:44,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:45,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:46,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:46,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:47,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:48,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:49,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:50,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:51,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:51,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:52,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:53,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:53,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:02:53,057 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:02:53,058 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:02:53,059 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:02:53,498 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:02:54,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:54,500 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:02:55,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:56,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:57,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:57,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:02:57,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:02:57,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:02:58,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:02:59,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:00,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:01,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:02,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:02,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:03,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:04,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:04,495 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:03:05,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:06,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:07,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:07,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:08,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:09,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:10,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:11,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:12,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:12,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:03:12,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:03:13,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:13,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:14,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:15,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:16,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:17,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:18,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:18,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:19,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:20,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:20,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:03:20,406 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:03:20,407 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:03:20,408 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:03:20,527 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:03:21,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:22,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:22,528 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:03:23,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:23,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:24,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:25,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:26,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:27,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:27,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:03:27,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:03:28,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:29,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:29,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:30,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:31,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:32,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:33,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:34,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:34,499 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:03:35,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:35,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:36,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:37,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:38,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:39,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:40,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:40,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:41,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:42,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:42,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:03:42,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:03:43,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:44,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:45,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:45,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:46,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:47,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:48,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:49,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:50,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:50,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:03:50,255 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:03:50,256 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:03:50,257 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:03:50,551 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:03:51,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:51,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:52,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:52,554 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:03:53,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:54,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:55,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:56,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:56,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:03:57,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:57,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:03:57,545 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:03:58,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:03:59,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:00,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:01,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:02,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:02,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:03,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:04,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:04,502 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:04:05,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:06,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:07,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:07,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:08,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:09,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:10,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:11,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:12,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:12,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:04:12,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:04:13,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:13,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:14,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:15,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:16,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:17,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:18,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:18,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:19,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:20,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:21,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:21,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:04:21,668 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:04:21,668 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:04:21,669 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:04:22,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:22,581 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:04:22,582 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:04:23,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:24,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:24,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:25,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:26,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:27,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:27,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:04:27,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:04:28,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:29,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:30,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:30,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:31,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:32,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:33,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:34,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:34,506 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:04:35,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:36,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:36,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:37,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:38,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:39,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:40,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:41,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:41,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:42,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:42,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:04:42,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:04:43,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:44,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:45,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:46,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:47,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:47,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:48,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:49,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:50,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:51,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:52,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:53,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:04:53,175 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:04:53,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:04:53,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:53,177 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:04:53,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:53,608 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:04:54,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:54,610 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:04:55,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:56,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:57,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:57,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:04:57,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:04:58,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:04:58,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:04:59,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:00,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:01,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:02,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:03,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:04,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:04,510 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:05:04,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:05,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:06,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:07,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:08,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:09,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:09,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:10,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:11,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:12,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:12,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:05:12,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:05:13,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:14,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:15,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:15,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:16,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:17,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:18,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:19,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:20,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:20,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:21,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:22,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:23,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:23,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:05:23,440 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:05:23,440 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:05:23,442 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:05:23,641 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:05:24,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:24,643 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:05:25,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:26,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:26,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:27,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:27,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:05:27,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:05:28,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:29,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:30,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:31,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:31,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:32,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:33,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:34,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:34,514 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:05:35,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:36,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:37,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:37,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:38,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:39,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:40,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:41,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:42,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:42,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:05:42,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:05:42,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:43,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:44,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:45,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:46,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:47,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:47,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:48,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:49,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:50,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:51,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:52,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:52,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:53,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:54,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:05:54,320 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:05:54,320 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:05:54,322 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:05:54,671 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:05:54,672 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:05:54,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:55,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:56,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:57,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:05:57,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:05:57,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:58,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:05:58,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:05:59,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:00,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:01,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:02,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:03,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:04,518 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:06:04,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:04,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:05,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:06,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:07,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:08,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:09,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:09,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:10,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:11,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:12,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:06:12,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:06:12,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:13,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:14,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:15,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:15,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:16,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:17,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:18,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:19,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:20,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:20,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:21,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:22,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:23,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:24,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:25,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:25,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:26,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:06:26,264 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:06:26,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:06:26,266 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:06:26,701 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:06:26,702 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:06:26,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:27,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:06:27,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:06:27,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:28,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:29,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:30,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:31,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:31,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:32,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:33,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:34,523 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:06:34,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:35,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:36,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:37,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:37,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:38,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:39,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:40,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:41,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:42,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:06:42,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:06:42,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:42,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:43,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:44,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:45,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:46,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:47,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:47,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:48,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:49,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:50,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:51,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:52,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:52,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:53,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:54,800 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:55,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:56,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:56,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:06:56,985 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:06:56,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:06:56,987 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:06:57,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:06:57,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:06:57,729 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:06:57,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:58,731 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:06:58,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:06:58,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:06:59,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:00,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:01,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:02,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:03,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:04,527 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:07:04,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:04,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:05,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:06,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:07,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:08,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:09,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:09,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:11,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:12,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:12,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:07:12,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:07:13,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:14,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:14,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:15,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:16,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:17,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:18,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:19,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:20,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:20,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:21,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:22,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:23,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:24,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:25,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:25,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:26,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:27,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:27,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:07:27,553 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:07:28,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:28,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:07:28,744 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:07:28,744 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:07:28,745 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:07:28,757 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:07:29,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:30,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:31,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:31,759 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:07:31,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:32,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:33,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:34,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:34,531 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:07:35,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:36,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:37,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:37,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:38,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:39,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:40,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:41,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:42,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:42,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:07:42,551 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:07:42,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:43,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:44,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:45,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:46,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:47,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:47,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:48,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:49,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:50,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:51,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:52,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:52,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:53,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:54,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:55,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:56,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:57,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:57,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:07:57,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:07:57,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:07:58,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:58,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:07:58,373 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:07:58,374 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:07:58,376 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:07:58,783 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:07:59,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:07:59,784 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:08:00,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:01,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:02,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:02,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:03,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:04,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:04,535 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:08:05,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:06,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:07,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:08,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:08,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:09,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:10,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:11,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:12,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:12,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:08:12,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:08:13,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:13,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:14,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:15,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:16,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:17,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:18,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:19,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:19,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:20,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:21,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:22,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:23,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:24,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:24,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:25,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:26,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:27,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:27,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:08:27,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:08:27,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:08:27,931 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:08:27,931 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:08:27,932 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:08:28,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:28,807 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:08:29,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:29,807 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:08:30,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:30,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:31,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:32,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:33,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:34,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:34,539 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:08:35,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:35,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:36,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:37,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:38,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:39,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:40,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:41,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:41,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:42,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:08:42,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:08:42,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:43,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:44,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:45,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:46,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:46,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:47,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:48,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:49,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:50,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:51,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:51,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:52,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:53,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:54,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:55,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:56,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:08:56,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:57,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:08:57,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:08:57,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:58,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:08:59,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:08:59,581 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:08:59,581 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:08:59,583 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:08:59,839 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:08:59,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:00,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:01,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:01,841 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:09:01,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:02,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:03,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:04,544 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:09:04,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:05,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:06,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:07,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:07,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:08,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:09,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:10,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:11,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:12,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:09:12,627 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:09:12,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:12,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:13,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:14,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:15,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:16,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:17,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:18,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:18,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:19,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:20,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:21,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:22,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:23,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:24,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:25,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:25,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:26,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:27,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:09:27,632 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:09:27,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:28,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:29,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:29,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:30,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:09:30,160 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:09:30,161 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:09:30,162 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:09:30,867 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:09:30,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:31,868 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:09:31,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:32,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:33,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:34,547 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:09:34,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:34,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:35,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:36,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:37,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:38,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:39,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:40,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:40,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:41,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:42,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:09:42,629 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:09:42,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:43,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:44,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:45,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:45,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:46,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:47,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:48,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:49,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:50,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:50,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:51,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:52,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:53,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:54,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:56,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:56,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:09:57,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:57,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:09:57,641 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:09:58,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:09:59,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:00,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:01,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:01,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:01,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:10:01,729 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:10:01,730 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:10:01,731 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:10:01,898 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:10:02,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:03,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:03,899 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:10:04,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:04,552 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:10:05,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:06,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:06,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:07,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:08,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:09,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:10,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:11,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:12,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:12,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:12,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:10:12,635 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:10:13,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:14,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:15,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:16,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:17,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:18,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:18,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:19,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:20,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:21,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:22,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:23,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:24,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:24,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:25,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:26,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:27,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:27,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:10:27,652 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:10:28,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:29,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:29,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:30,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:31,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:32,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:33,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:33,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:10:33,416 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:10:33,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:10:33,418 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:10:33,928 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:10:34,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:34,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:34,556 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:10:35,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:35,929 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:10:36,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:37,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:38,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:39,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:40,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:40,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:41,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:42,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:42,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:10:42,639 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:10:43,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:44,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:45,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:46,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:46,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:47,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:48,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:49,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:50,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:51,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:52,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:52,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:53,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:54,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:55,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:56,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:57,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:10:57,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:57,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:10:57,638 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:10:58,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:10:59,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:00,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:01,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:02,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:02,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:03,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:04,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:04,564 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:11:05,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:06,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:07,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:07,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:11:07,509 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:11:07,510 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:11:07,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:07,511 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:11:07,962 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:11:08,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:09,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:09,963 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:11:10,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:11,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:12,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:12,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:11:12,638 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:11:12,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:13,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:14,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:15,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:16,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:17,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:18,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:18,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:19,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:20,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:21,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:22,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:23,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:23,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:24,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:25,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:26,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:27,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:27,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:11:27,638 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:11:28,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:29,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:29,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:30,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:31,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:32,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:33,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:34,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:34,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:34,565 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:11:35,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:36,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:37,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:38,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:39,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:39,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:40,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:41,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:41,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:11:41,632 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:11:41,633 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:11:41,638 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:11:41,992 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:11:42,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:42,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:11:42,638 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:11:43,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:43,994 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:11:44,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:45,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:45,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:46,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:47,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:48,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:49,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:50,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:50,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:51,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:52,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:53,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:54,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:55,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:56,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:11:56,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:57,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:11:57,639 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:11:57,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:58,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:11:59,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:00,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:01,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:01,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:02,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:03,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:04,570 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:12:04,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:05,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:06,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:06,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:07,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:08,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:09,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:10,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:11,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:12,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:12,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:12:12,646 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:12:12,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:13,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:14,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:15,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:12:15,098 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:12:15,098 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:12:15,104 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:12:15,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:16,027 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:12:16,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:17,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:17,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:18,030 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:12:18,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:19,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:20,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:21,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:22,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:23,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:23,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:24,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:25,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:26,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:27,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:12:27,643 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:12:27,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:28,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:28,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:29,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:30,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:31,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:32,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:33,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:34,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:34,573 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:12:34,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:35,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:36,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:37,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:38,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:39,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:39,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:40,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:41,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:42,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:12:42,646 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:12:42,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:43,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:44,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:44,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:45,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:12:45,687 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:12:45,688 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:12:45,693 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:12:46,054 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:12:46,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:47,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:48,056 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:12:48,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:49,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:50,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:50,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:51,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:52,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:53,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:54,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:55,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:56,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:56,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:12:57,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:57,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:12:57,645 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:12:58,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:12:59,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:00,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:01,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:01,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:02,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:03,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:04,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:04,577 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:13:05,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:06,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:06,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:07,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:08,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:09,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:10,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:11,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:12,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:12,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:12,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:13:12,650 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:13:13,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:14,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:15,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:16,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:16,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:13:16,214 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:13:16,214 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:13:16,219 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:13:17,083 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:13:17,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:17,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:18,084 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:13:18,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:19,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:20,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:21,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:22,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:23,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:23,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:24,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:25,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:26,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:27,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:27,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:13:27,645 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:13:28,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:29,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:29,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:30,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:31,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:32,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:33,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:34,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:34,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:34,581 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:13:35,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:36,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:37,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:38,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:39,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:40,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:40,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:41,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:42,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:42,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:13:42,660 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:13:43,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:44,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:45,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:45,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:46,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:13:46,137 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:13:46,137 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:13:46,138 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:13:46,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:47,107 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:13:47,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:48,107 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:13:48,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:49,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:50,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:51,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:51,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:52,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:53,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:54,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:55,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:56,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:57,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:13:57,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:57,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:13:57,651 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:13:58,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:13:59,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:00,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:01,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:02,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:02,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:03,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:04,586 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:14:04,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:05,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:06,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:07,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:07,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:08,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:09,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:10,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:11,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:12,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:12,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:14:12,660 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:14:12,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:13,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:14,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:15,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:16,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:17,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:18,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:18,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:14:18,344 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:14:18,345 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:14:18,347 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:14:18,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:19,136 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:14:19,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:20,137 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:14:20,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:21,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:22,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:23,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:24,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:24,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:25,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:26,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:27,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:14:27,654 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:14:27,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:28,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:29,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:30,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:30,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:31,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:32,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:33,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:34,591 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:14:34,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:35,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:35,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:36,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:37,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:38,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:39,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:40,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:41,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:41,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:42,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:14:42,662 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:14:42,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:43,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:44,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:45,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:14:45,411 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:14:45,412 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:14:45,413 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:14:45,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:46,162 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:14:46,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:46,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:47,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:48,164 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:14:48,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:49,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:50,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:51,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:52,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:52,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:53,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:54,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:55,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:56,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:57,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:14:57,655 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:14:57,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:14:57,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:58,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:14:59,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:00,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:01,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:02,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:03,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:03,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:04,594 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:15:04,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:05,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:06,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:07,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:08,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:08,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:09,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:10,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:11,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:12,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:15:12,663 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:15:12,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:13,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:13,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:14,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:15:14,629 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:15:14,630 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:15:14,631 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:15:15,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:15,188 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:15:16,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:16,189 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:15:17,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:18,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:19,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:19,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:20,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:21,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:22,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:23,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:24,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:24,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:25,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:26,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:27,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:27,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:15:27,660 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:15:28,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:29,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:30,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:30,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:31,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:32,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:33,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:34,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:34,597 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:15:35,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:35,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:36,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:37,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:38,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:39,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:40,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:41,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:41,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:42,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:42,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:15:42,666 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:15:43,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:44,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:15:44,051 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:15:44,052 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:15:44,054 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:15:44,218 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:15:44,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:45,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:46,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:46,220 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:15:47,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:47,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:48,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:49,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:50,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:51,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:52,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:52,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:53,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:54,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:55,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:56,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:57,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:57,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:15:57,664 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:15:57,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:15:58,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:15:59,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:00,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:01,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:02,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:03,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:03,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:04,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:04,601 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:16:05,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:06,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:07,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:08,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:08,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:09,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:10,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:11,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:12,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:12,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:16:12,672 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:16:13,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:14,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:14,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:14,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:16:14,340 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:16:14,341 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:16:14,342 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:16:15,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:15,250 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:16:16,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:16,251 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:16:17,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:18,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:19,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:19,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:20,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:21,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:22,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:23,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:24,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:25,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:25,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:26,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:27,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:27,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:16:27,670 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:16:28,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:29,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:30,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:30,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:31,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:32,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:33,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:34,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:34,605 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:16:35,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:35,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:36,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:37,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:38,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:39,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:40,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:41,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:41,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:41,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:16:41,646 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:16:41,646 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:16:41,647 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:16:42,276 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:16:42,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:42,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:16:42,674 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:16:43,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:44,278 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:16:44,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:45,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:46,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:47,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:47,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:48,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:49,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:50,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:51,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:52,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:53,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:53,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:54,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:55,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:56,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:57,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:16:57,670 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:16:57,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:58,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:16:58,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:16:59,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:00,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:01,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:02,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:03,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:03,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:04,609 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:17:04,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:05,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:06,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:07,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:08,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:08,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:09,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:10,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:11,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:12,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:17:12,671 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:17:12,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:12,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:17:12,898 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:17:12,898 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:17:12,900 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:17:13,311 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:17:13,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:13,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:14,312 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:17:14,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:15,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:16,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:17,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:18,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:19,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:19,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:20,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:21,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:22,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:23,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:24,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:25,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:25,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:26,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:27,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:17:27,671 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:17:27,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:28,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:29,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:30,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:30,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:31,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:32,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:33,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:34,614 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:17:34,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:35,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:35,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:36,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:37,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:38,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:39,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:40,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:41,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:41,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:42,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:17:42,692 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:17:42,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:43,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:44,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:45,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:46,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:46,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:47,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:17:47,023 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:17:47,023 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:17:47,025 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:17:47,339 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:17:47,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:48,340 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:17:48,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:49,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:50,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:51,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:51,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:52,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:53,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:54,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:55,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:56,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:57,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:17:57,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:17:57,677 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:17:57,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:58,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:17:59,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:00,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:01,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:02,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:02,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:03,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:04,618 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:18:04,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:05,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:06,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:07,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:07,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:08,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:09,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:10,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:11,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:12,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:18:12,695 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:18:12,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:12,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:13,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:14,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:15,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:16,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:17,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:18,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:18,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:18:18,423 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:18:18,423 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:18:18,425 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:18:18,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:19,381 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:18:19,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:20,382 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:18:20,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:21,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:22,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:23,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:23,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:24,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:25,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:26,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:27,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:18:27,681 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:18:27,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:28,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:29,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:29,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:30,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:31,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:32,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:33,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:34,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:34,622 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:18:34,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:35,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:36,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:37,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:38,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:39,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:40,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:40,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:41,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:42,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:18:42,681 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:18:42,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:43,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:44,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:45,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:45,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:46,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:47,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:48,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:49,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:18:49,193 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:18:49,193 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:18:49,195 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:18:49,416 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:18:49,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:50,417 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:18:50,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:51,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:51,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:52,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:53,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:54,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:55,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:56,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:18:56,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:57,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:18:57,681 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:18:57,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:58,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:18:59,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:00,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:01,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:01,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:02,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:03,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:04,626 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:19:04,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:05,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:06,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:06,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:07,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:08,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:09,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:10,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:11,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:12,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:12,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:19:12,688 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:19:12,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:13,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:14,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:15,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:16,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:17,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:17,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:19:17,778 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:19:17,778 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:19:17,780 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:19:18,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:18,444 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:19:19,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:20,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:20,446 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:19:21,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:22,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:23,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:23,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:24,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:25,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:26,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:27,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:27,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:19:27,684 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:19:28,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:29,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:29,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:30,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:31,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:32,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:33,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:34,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:34,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:34,630 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:19:35,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:36,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:37,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:38,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:39,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:40,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:40,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:41,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:42,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:42,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:19:42,685 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:19:43,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:44,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:45,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:45,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:46,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:47,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:47,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:19:47,969 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:19:47,970 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:19:47,971 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:19:48,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:48,475 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:19:49,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:50,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:50,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:50,476 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:19:51,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:52,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:53,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:54,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:55,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:56,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:56,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:19:57,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:57,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:19:57,686 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:19:58,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:19:59,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:00,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:01,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:02,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:02,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:03,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:04,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:04,635 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:20:05,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:06,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:07,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:07,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:08,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:09,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:10,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:11,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:12,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:12,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:20:12,685 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:20:12,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:13,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:14,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:15,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:16,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:17,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:17,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:20:17,869 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:20:17,869 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:20:17,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:17,870 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:20:18,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:18,500 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:20:19,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:20,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:20,502 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:20:21,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:22,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:23,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:23,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:24,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:25,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:26,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:27,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:27,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:20:27,686 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:20:28,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:28,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:29,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:30,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:31,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:32,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:33,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:34,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:34,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:34,638 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:20:35,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:36,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:37,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:38,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:39,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:40,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:40,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:41,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:42,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:42,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:20:42,688 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:20:43,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:44,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:45,414 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:45,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:46,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:47,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:47,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:20:47,865 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:20:47,865 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:20:47,866 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:20:48,524 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:20:48,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:49,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:50,525 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:20:50,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:51,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:51,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:52,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:53,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:54,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:55,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:56,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:57,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:20:57,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:57,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:20:57,688 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:20:58,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:20:59,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:00,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:01,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:02,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:02,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:03,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:04,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:04,643 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:21:05,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:06,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:07,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:07,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:08,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:09,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:10,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:11,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:12,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:12,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:21:12,697 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:21:12,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:13,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:14,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:15,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:16,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:16,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:21:16,902 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:21:16,902 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:21:16,903 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:21:17,553 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:21:17,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:17,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:18,554 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:21:18,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:19,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:20,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:21,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:22,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:23,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:23,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:24,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:25,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:26,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:27,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:27,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:21:27,690 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:21:28,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:28,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:29,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:30,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:31,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:32,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:33,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:34,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:34,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:34,647 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:21:35,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:36,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:37,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:38,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:39,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:39,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:40,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:41,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:42,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:42,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:21:42,690 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:21:43,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:44,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:44,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:45,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:46,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:21:46,176 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:21:46,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:21:46,177 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:21:46,584 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:21:46,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:47,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:48,585 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:21:48,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:49,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:50,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:50,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:51,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:52,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:53,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:54,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:55,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:56,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:21:56,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:57,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:57,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:21:57,690 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:21:58,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:21:59,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:00,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:01,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:01,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:02,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:03,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:04,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:04,650 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:22:05,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:06,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:06,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:07,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:08,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:09,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:10,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:11,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:12,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:12,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:12,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:22:12,695 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:22:13,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:14,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:15,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:16,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:16,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:22:16,740 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:22:16,740 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:22:16,741 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:22:17,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:17,610 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:22:17,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:18,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:18,612 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:22:19,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:20,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:21,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:22,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:23,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:23,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:24,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:25,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:26,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:27,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:27,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:22:27,692 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:22:28,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:29,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:29,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:30,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:31,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:32,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:33,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:34,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:34,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:34,655 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:22:35,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:36,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:37,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:38,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:39,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:40,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:40,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:41,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:42,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:42,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:22:42,704 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:22:43,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:44,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:45,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:45,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:46,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:47,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:22:47,590 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:22:47,590 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:22:47,591 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:22:47,644 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:22:48,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:48,646 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:22:48,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:49,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:50,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:51,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:51,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:52,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:53,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:54,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:55,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:56,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:22:56,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:57,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:22:57,697 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:22:57,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:58,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:22:59,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:00,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:01,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:02,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:02,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:03,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:04,659 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:23:04,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:05,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:06,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:07,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:07,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:08,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:09,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:10,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:11,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:12,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:23:12,699 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:23:12,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:12,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:13,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:14,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:15,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:16,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:16,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:23:16,984 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:23:16,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:23:16,986 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:23:17,690 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:23:17,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:18,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:18,691 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:23:18,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:19,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:20,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:21,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:22,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:23,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:23,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:24,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:25,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:26,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:27,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:23:27,699 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:23:27,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:28,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:29,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:29,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:30,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:31,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:32,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:33,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:34,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:34,663 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:23:34,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:35,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:36,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:37,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:38,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:39,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:40,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:40,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:41,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:42,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:23:42,700 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:23:42,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:43,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:44,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:45,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:45,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:46,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:23:46,343 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:23:46,344 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:23:46,345 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:23:46,720 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:23:46,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:47,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:48,722 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:23:48,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:49,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:50,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:51,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:51,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:52,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:53,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:54,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:55,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:56,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:57,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:23:57,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:23:57,700 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:23:57,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:58,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:23:59,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:00,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:01,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:02,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:03,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:03,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:04,667 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:24:04,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:05,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:06,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:07,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:08,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:08,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:09,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:10,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:11,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:12,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:24:12,707 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:24:12,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:13,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:13,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:14,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:15,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:16,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:17,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:18,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:24:18,510 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:24:18,510 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:24:18,511 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:24:18,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:18,753 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:24:18,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:19,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:20,754 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:24:20,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:21,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:22,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:23,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:23,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:24,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:25,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:26,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:27,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:24:27,705 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:24:27,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:28,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:29,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:29,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:30,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:31,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:32,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:33,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:34,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:34,671 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:24:34,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:35,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:36,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:37,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:38,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:39,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:40,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:40,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:41,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:42,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:24:42,705 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:24:42,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:43,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:44,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:45,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:45,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:46,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:47,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:48,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:49,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:24:49,106 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:24:49,107 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:24:49,108 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:24:49,781 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:24:49,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:50,782 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:24:50,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:51,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:51,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:52,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:53,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:54,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:55,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:56,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:24:56,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:57,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:24:57,705 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:24:57,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:58,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:24:59,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:00,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:01,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:02,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:02,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:03,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:04,676 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:25:04,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:05,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:06,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:07,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:07,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:08,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:09,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:10,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:11,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:12,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:25:12,705 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:25:12,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:12,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:13,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:14,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:15,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:16,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:17,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:18,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:18,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:19,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:25:19,893 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:25:19,894 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:25:19,896 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:25:20,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:20,810 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:25:20,811 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:25:21,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:22,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:23,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:24,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:24,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:25,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:26,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:27,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:27,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:25:27,705 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:25:28,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:29,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:30,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:30,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:31,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:32,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:33,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:34,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:34,680 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:25:35,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:35,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:36,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:37,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:38,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:39,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:40,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:41,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:41,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:42,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:42,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:25:42,705 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:25:43,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:44,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:45,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:46,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:46,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:47,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:48,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:49,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:50,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:50,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:25:50,998 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:25:50,999 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:25:51,000 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:25:51,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:51,835 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:25:52,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:52,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:52,836 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:25:53,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:54,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:55,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:56,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:57,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:57,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:25:57,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:25:57,706 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:25:58,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:25:59,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:00,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:01,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:02,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:03,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:03,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:04,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:04,683 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:26:05,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:06,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:07,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:08,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:09,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:09,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:10,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:11,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:12,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:12,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:26:12,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:26:13,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:14,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:14,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:15,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:16,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:17,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:26:17,417 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:26:17,418 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:26:17,419 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:26:17,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:17,864 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:26:18,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:18,866 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:26:19,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:20,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:20,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:21,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:22,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:23,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:24,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:25,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:26,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:26,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:27,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:26:27,710 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:26:27,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:28,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:29,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:30,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:31,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:32,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:32,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:33,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:34,687 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:26:34,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:35,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:36,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:37,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:37,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:38,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:39,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:40,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:41,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:42,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:26:42,713 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:26:42,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:42,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:43,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:44,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:45,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:46,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:47,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:26:47,423 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:26:47,424 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:26:47,425 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:26:47,895 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:26:47,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:48,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:48,898 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:26:48,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:49,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:50,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:51,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:52,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:53,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:53,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:54,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:55,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:56,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:57,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:26:57,713 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:26:57,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:58,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:26:59,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:26:59,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:00,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:01,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:02,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:03,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:04,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:04,691 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:27:04,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:05,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:06,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:07,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:08,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:09,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:09,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:10,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:11,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:12,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:27:12,713 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:27:12,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:13,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:14,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:15,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:15,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:16,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:17,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:18,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:19,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:20,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:20,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:21,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:27:21,580 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:27:21,581 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:27:21,583 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:27:21,924 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:27:21,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:22,925 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:27:22,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:23,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:24,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:25,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:26,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:26,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:27,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:27:27,713 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:27:27,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:28,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:29,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:30,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:31,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:32,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:32,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:34,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:34,696 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:27:35,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:36,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:37,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:37,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:38,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:39,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:40,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:41,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:42,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:42,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:27:42,713 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:27:42,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:43,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:44,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:45,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:46,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:47,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:48,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:48,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:49,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:50,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:51,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:51,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:27:51,717 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:27:51,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:27:51,720 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:27:51,952 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:27:52,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:52,953 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:27:53,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:54,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:54,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:27:55,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:56,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:57,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:57,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:27:57,713 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:27:58,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:27:59,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:00,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:00,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:01,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:02,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:03,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:04,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:04,700 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:28:05,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:05,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:06,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:07,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:08,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:09,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:10,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:11,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:11,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:12,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:12,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:28:12,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:28:13,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:14,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:15,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:16,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:16,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:17,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:18,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:19,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:20,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:21,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:21,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:22,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:22,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:28:22,488 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:28:22,489 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:28:22,493 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:28:22,980 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:28:23,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:24,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:24,981 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:28:25,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:26,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:27,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:27,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:27,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:28:27,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:28:28,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:29,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:30,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:31,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:32,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:33,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:33,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:34,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:34,704 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:28:35,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:36,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:37,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:38,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:38,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:39,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:40,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:41,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:42,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:42,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:28:42,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:28:43,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:44,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:44,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:45,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:46,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:47,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:48,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:49,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:49,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:50,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:51,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:52,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:53,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:54,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:54,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:28:54,367 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:28:54,367 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:28:54,368 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:28:55,009 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:28:55,010 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:28:55,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:55,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:28:56,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:57,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:57,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:28:57,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:28:58,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:28:59,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:00,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:00,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:01,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:02,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:03,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:04,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:04,708 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:29:05,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:05,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:06,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:07,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:08,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:09,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:10,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:11,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:11,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:12,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:12,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:29:12,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:29:13,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:14,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:15,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:16,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:17,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:17,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:18,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:19,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:20,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:21,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:22,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:22,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:23,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:24,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:25,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:25,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:29:25,988 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:29:25,988 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:29:25,989 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:29:26,043 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:29:26,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:27,045 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:29:27,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:27,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:29:27,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:29:27,789 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:28,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:29,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:30,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:31,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:32,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:33,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:33,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:34,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:34,712 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:29:35,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:36,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:37,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:38,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:39,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:39,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:40,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:41,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:42,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:42,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:29:42,715 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:29:43,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:44,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:45,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:45,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:46,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:47,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:48,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:49,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:50,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:50,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:51,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:52,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:53,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:54,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:55,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:55,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:29:56,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:57,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:57,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:29:57,716 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:29:58,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:29:59,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:00,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:00,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:01,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:30:01,161 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:30:01,161 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:30:01,162 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:30:01,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:02,072 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:30:02,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:03,073 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:30:03,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:04,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:04,716 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:30:05,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:05,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:06,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:07,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:08,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:09,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:10,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:11,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:11,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:12,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:12,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:30:12,897 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:30:13,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:14,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:15,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:16,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:16,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:17,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:18,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:19,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:20,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:21,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:22,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:22,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:23,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:24,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:25,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:26,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:27,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:27,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:27,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:30:27,807 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:30:28,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:29,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:29,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:30:29,632 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:30:29,633 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:30:29,634 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:30:30,104 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:30:30,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:31,105 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:30:31,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:32,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:33,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:33,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:34,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:34,719 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:30:35,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:36,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:37,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:38,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:38,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:39,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:40,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:41,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:42,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:42,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:30:42,803 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:30:43,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:43,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:44,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:45,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:46,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:47,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:48,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:49,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:49,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:50,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:51,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:52,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:53,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:54,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:54,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:30:55,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:56,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:57,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:57,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:30:57,809 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:30:58,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:30:59,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:30:59,484 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:30:59,484 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:30:59,485 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:30:59,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:00,138 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:31:00,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:00,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:01,139 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:31:01,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:02,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:03,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:04,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:04,724 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:31:05,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:05,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:06,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:07,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:08,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:09,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:10,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:11,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:11,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:12,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:12,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:31:12,807 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:31:13,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:14,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:15,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:16,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:16,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:17,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:18,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:19,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:20,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:21,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:22,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:22,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:23,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:24,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:25,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:26,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:27,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:27,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:27,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:31:27,813 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:31:28,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:29,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:29,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:31:29,715 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:31:29,715 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:31:29,717 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:31:30,167 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:31:30,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:31,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:32,169 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:31:32,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:32,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:33,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:34,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:34,727 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:31:35,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:36,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:37,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:38,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:38,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:39,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:40,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:41,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:42,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:42,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:31:42,810 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:31:43,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:44,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:44,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:45,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:46,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:47,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:48,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:49,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:49,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:50,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:51,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:52,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:53,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:54,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:54,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:31:55,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:56,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:57,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:57,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:31:57,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:31:58,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:59,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:31:59,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:00,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:32:00,252 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:32:00,253 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:32:00,254 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:32:00,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:01,192 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:32:01,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:02,193 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:32:02,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:03,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:04,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:04,731 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:32:04,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:05,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:06,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:07,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:08,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:09,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:10,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:10,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:11,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:12,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:12,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:32:12,813 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:32:13,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:14,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:15,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:16,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:16,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:17,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:18,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:19,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:20,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:21,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:21,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:22,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:23,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:24,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:25,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:26,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:27,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:27,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:27,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:32:27,821 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:32:28,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:29,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:30,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:31,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:32,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:32,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:32,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:32:32,873 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:32:32,873 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:32:32,874 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:32:33,222 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:32:33,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:34,222 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:32:34,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:34,735 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:32:35,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:36,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:37,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:37,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:38,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:39,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:40,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:41,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:42,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:42,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:32:42,818 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:32:42,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:43,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:44,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:45,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:46,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:47,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:48,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:48,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:49,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:50,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:51,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:52,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:53,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:53,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:54,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:55,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:56,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:57,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:32:57,818 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:32:57,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:58,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:32:59,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:32:59,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:00,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:01,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:02,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:03,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:04,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:04,743 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:33:04,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:05,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:06,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:33:06,062 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:33:06,063 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:33:06,064 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:33:06,256 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:33:06,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:07,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:08,258 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:33:08,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:09,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:10,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:10,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:11,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:12,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:33:12,818 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:33:12,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:13,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:14,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:15,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:16,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:16,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:17,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:19,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:20,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:21,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:22,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:22,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:23,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:24,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:25,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:26,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:27,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:27,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:27,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:33:27,818 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:33:28,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:29,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:30,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:31,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:32,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:32,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:33,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:34,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:34,753 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:33:35,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:36,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:36,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:33:36,668 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:33:36,669 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:33:36,670 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:33:37,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:37,289 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:33:38,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:38,292 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:33:38,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:39,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:40,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:41,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:42,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:42,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:33:42,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:33:43,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:43,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:44,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:45,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:46,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:47,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:48,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:49,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:49,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:50,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:51,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:52,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:53,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:54,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:54,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:33:55,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:56,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:57,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:57,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:33:57,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:33:58,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:33:59,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:00,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:00,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:01,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:02,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:03,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:04,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:04,756 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:34:05,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:05,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:06,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:07,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:08,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:08,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:34:08,793 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:34:08,794 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:34:08,795 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:34:09,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:09,317 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:34:10,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:10,318 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:34:10,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:11,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:12,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:12,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:34:12,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:34:13,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:14,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:15,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:16,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:16,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:17,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:18,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:19,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:20,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:21,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:22,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:22,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:23,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:24,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:25,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:26,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:27,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:27,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:27,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:34:27,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:34:28,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:29,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:30,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:31,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:32,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:32,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:33,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:34,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:34,756 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:34:35,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:36,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:37,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:37,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:38,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:38,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:34:38,841 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:34:38,841 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:34:38,846 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:34:39,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:39,339 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:34:40,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:40,340 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:34:41,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:42,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:42,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:34:42,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:34:42,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:43,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:44,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:45,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:46,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:47,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:48,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:48,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:49,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:50,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:51,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:52,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:53,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:54,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:54,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:34:55,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:56,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:57,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:57,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:34:57,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:34:58,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:59,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:34:59,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:00,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:01,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:02,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:03,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:04,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:04,760 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:35:04,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:05,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:06,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:07,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:08,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:09,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:10,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:10,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:11,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:12,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:12,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:35:12,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:35:13,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:13,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:35:13,386 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:35:13,387 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:35:13,392 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:35:14,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:14,369 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:35:14,369 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:35:15,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:15,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:16,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:17,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:18,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:19,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:20,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:21,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:21,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:22,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:23,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:24,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:25,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:26,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:27,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:27,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:27,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:35:27,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:35:28,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:29,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:30,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:31,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:32,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:32,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:33,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:34,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:34,765 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:35:35,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:36,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:37,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:38,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:38,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:39,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:40,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:41,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:42,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:42,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:35:42,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:35:42,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:35:42,992 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:35:42,992 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:35:42,994 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:35:43,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:43,392 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:35:43,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:44,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:44,394 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:35:45,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:46,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:47,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:48,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:49,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:49,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:50,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:51,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:52,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:53,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:54,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:55,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:55,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:35:56,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:57,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:57,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:35:57,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:35:58,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:35:59,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:00,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:00,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:01,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:02,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:03,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:04,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:04,770 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:36:05,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:06,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:06,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:07,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:08,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:09,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:10,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:11,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:11,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:12,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:12,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:36:12,824 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:36:13,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:36:13,212 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:36:13,212 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:36:13,214 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:36:13,419 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:36:13,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:14,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:14,419 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:36:15,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:16,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:17,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:17,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:18,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:19,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:20,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:21,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:22,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:23,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:23,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:24,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:25,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:26,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:27,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:27,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:36:27,824 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:36:28,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:28,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:29,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:30,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:31,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:32,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:33,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:33,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:34,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:34,773 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:36:35,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:36,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:37,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:38,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:38,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:39,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:40,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:41,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:42,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:42,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:36:42,831 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:36:43,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:44,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:44,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:36:44,603 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:36:44,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:36:44,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:44,610 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:36:45,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:45,449 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:36:46,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:46,451 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:36:47,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:48,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:49,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:49,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:50,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:51,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:52,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:53,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:54,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:55,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:55,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:36:56,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:57,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:57,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:36:57,827 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:36:58,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:36:59,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:00,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:00,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:01,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:02,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:03,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:04,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:04,777 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:37:05,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:06,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:06,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:07,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:08,390 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:09,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:10,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:11,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:11,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:12,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:12,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:37:12,827 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:37:13,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:14,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:15,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:37:15,328 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:37:15,328 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:37:15,333 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:37:15,479 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:37:15,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:16,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:16,481 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:37:17,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:17,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:18,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:19,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:20,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:21,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:22,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:22,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:23,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:24,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:25,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:26,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:27,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:27,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:37:27,827 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:37:27,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:28,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:29,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:30,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:31,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:32,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:33,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:33,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:34,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:34,780 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:37:35,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:36,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:37,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:38,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:38,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:39,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:40,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:41,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:42,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:42,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:37:42,832 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:37:43,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:43,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:44,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:45,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:46,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:37:46,393 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:37:46,394 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:37:46,399 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:37:46,508 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:37:46,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:47,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:48,510 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:37:48,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:49,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:49,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:50,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:51,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:52,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:53,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:54,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:55,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:55,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:37:56,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:57,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:57,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:37:57,829 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:37:58,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:37:59,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:00,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:01,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:01,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:02,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:03,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:04,784 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:38:04,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:05,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:06,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:07,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:07,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:08,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:09,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:10,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:11,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:12,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:12,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:12,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:38:12,834 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:38:13,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:14,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:15,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:16,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:17,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:38:17,308 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:38:17,308 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:38:17,314 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:38:17,537 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:38:17,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:18,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:18,538 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:38:18,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:19,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:20,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:21,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:22,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:23,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:23,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:24,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:25,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:26,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:27,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:27,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:38:27,834 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:38:28,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:29,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:29,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:30,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:31,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:32,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:33,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:34,788 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:38:34,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:35,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:35,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:36,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:37,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:38,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:39,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:40,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:40,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:41,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:42,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:42,837 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:38:42,838 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:38:43,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:44,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:45,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:45,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:46,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:46,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:38:46,893 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:38:46,894 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:38:46,895 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:38:47,569 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:38:47,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:48,570 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:38:48,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:49,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:50,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:51,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:51,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:52,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:53,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:54,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:55,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:56,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:57,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:38:57,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:57,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:38:57,837 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:38:58,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:38:59,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:00,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:01,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:02,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:03,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:03,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:04,793 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:39:04,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:05,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:06,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:07,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:08,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:09,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:09,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:10,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:11,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:12,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:12,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:39:12,842 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:39:13,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:14,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:14,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:15,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:16,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:17,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:18,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:39:18,702 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:39:18,702 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:39:18,708 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:39:19,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:19,596 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:39:20,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:20,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:20,597 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:39:21,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:22,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:23,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:24,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:25,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:26,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:26,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:27,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:27,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:39:27,843 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:39:28,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:29,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:30,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:31,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:32,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:32,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:33,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:34,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:34,797 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:39:35,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:36,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:37,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:37,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:38,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:39,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:40,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:41,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:42,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:42,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:39:42,843 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:39:42,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:43,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:44,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:45,128 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:46,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:47,128 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:48,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:48,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:49,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:49,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:39:49,433 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:39:49,433 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:39:49,439 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:39:49,619 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:39:50,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:50,620 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:39:51,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:52,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:53,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:53,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:54,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:55,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:56,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:57,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:57,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:39:57,843 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:39:58,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:39:58,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:39:59,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:00,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:01,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:02,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:03,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:04,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:04,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:04,801 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:40:05,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:06,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:07,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:08,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:09,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:10,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:10,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:11,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:12,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:12,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:40:12,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:40:13,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:14,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:15,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:15,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:16,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:17,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:18,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:19,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:20,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:20,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:20,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:40:20,826 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:40:20,826 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:40:20,829 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:40:21,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:21,647 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:40:22,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:22,648 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:40:23,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:24,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:25,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:25,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:26,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:27,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:27,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:40:27,843 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:40:28,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:29,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:30,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:31,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:31,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:32,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:33,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:34,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:34,805 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:40:35,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:36,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:36,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:37,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:38,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:39,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:40,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:41,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:42,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:42,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:42,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:40:42,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:40:43,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:44,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:45,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:46,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:47,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:47,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:48,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:49,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:50,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:50,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:40:50,524 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:40:50,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:40:50,526 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:40:50,673 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:40:51,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:52,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:52,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:52,675 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:40:53,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:54,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:55,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:56,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:57,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:57,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:40:57,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:40:57,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:40:58,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:40:59,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:00,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:01,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:02,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:03,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:03,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:04,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:04,809 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:41:05,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:06,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:07,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:08,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:08,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:09,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:10,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:11,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:12,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:12,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:41:12,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:41:13,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:13,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:14,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:15,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:16,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:17,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:18,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:19,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:19,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:20,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:20,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:41:20,890 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:41:20,891 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:41:20,892 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:41:21,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:21,701 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:41:22,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:22,702 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:41:23,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:24,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:25,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:25,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:26,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:27,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:27,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:41:27,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:41:28,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:29,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:30,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:31,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:31,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:32,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:33,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:34,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:34,814 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:41:35,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:36,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:36,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:37,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:38,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:39,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:40,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:41,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:42,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:42,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:42,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:41:42,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:41:43,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:44,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:45,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:46,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:47,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:47,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:48,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:49,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:50,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:51,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:52,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:52,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:53,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:54,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:54,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:41:54,549 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:41:54,549 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:41:54,552 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:41:54,729 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:41:55,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:56,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:56,731 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:41:57,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:57,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:41:57,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:41:57,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:41:58,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:41:59,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:00,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:01,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:02,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:03,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:03,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:04,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:04,817 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:42:05,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:06,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:07,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:08,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:08,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:09,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:10,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:11,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:12,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:12,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:42:12,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:42:13,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:13,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:14,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:15,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:16,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:17,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:18,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:19,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:19,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:20,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:21,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:22,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:23,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:24,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:24,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:25,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:26,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:42:26,036 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:42:26,037 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:42:26,043 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:42:26,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:26,762 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:42:26,763 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:42:27,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:27,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:42:27,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:42:28,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:29,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:30,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:30,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:31,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:32,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:33,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:34,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:34,822 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:42:35,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:35,824 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:36,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:37,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:38,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:39,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:40,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:41,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:41,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:42,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:42,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:42:42,845 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:42:43,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:44,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:45,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:46,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:47,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:47,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:48,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:49,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:50,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:51,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:52,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:52,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:53,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:54,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:42:54,461 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:42:54,461 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:42:54,463 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:42:54,791 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:42:54,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:55,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:56,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:56,794 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:42:57,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:42:57,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:57,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:42:57,845 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:42:58,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:42:59,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:00,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:01,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:02,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:03,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:03,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:04,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:04,826 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:43:05,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:06,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:07,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:08,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:08,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:09,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:10,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:11,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:12,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:12,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:43:12,845 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:43:13,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:13,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:14,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:15,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:16,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:17,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:18,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:19,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:19,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:20,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:21,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:22,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:23,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:43:23,590 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:43:23,590 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:43:23,591 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:43:23,827 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:43:24,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:24,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:24,828 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:43:25,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:25,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:26,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:27,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:27,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:43:27,845 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:43:28,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:29,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:30,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:31,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:31,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:32,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:33,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:34,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:34,830 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:43:35,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:36,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:36,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:37,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:38,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:39,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:40,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:41,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:42,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:42,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:42,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:43:42,845 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:43:43,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:44,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:45,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:46,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:47,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:48,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:48,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:49,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:50,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:51,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:52,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:53,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:53,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:54,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:55,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:43:55,722 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:43:55,722 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:43:55,724 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:43:55,859 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:43:56,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:56,860 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:43:57,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:57,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:43:57,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:43:58,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:43:58,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:43:59,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:00,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:01,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:02,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:03,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:04,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:04,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:04,834 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:44:05,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:06,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:07,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:08,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:09,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:10,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:10,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:11,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:12,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:12,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:44:12,849 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:44:13,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:14,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:15,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:16,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:16,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:17,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:18,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:19,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:20,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:21,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:21,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:21,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:44:21,442 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:44:21,442 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:44:21,444 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:44:21,885 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:44:22,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:22,886 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:44:23,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:24,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:25,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:26,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:26,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:27,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:27,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:44:27,849 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:44:28,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:29,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:30,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:31,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:32,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:32,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:33,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:34,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:34,839 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:44:35,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:36,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:37,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:37,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:38,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:39,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:40,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:41,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:42,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:42,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:44:42,849 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:44:42,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:43,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:44,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:45,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:46,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:47,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:48,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:48,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:49,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:50,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:50,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:44:50,243 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:44:50,243 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:44:50,244 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:44:50,910 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:44:50,911 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:44:51,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:52,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:53,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:53,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:54,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:55,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:56,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:57,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:57,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:44:57,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:44:58,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:44:59,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:44:59,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:00,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:01,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:02,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:03,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:04,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:04,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:04,842 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:45:05,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:06,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:07,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:08,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:09,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:09,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:10,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:11,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:12,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:12,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:45:12,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:45:13,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:14,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:15,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:15,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:16,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:17,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:18,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:19,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:20,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:20,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:20,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:45:20,379 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:45:20,380 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:45:20,381 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:45:20,938 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:45:21,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:22,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:22,939 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:45:23,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:24,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:25,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:26,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:26,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:27,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:27,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:45:27,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:45:28,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:29,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:30,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:31,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:32,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:32,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:33,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:34,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:34,847 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:45:35,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:36,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:37,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:37,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:38,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:39,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:40,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:41,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:42,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:42,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:45:42,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:45:42,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:43,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:44,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:45,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:46,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:47,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:48,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:48,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:49,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:49,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:45:49,857 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:45:49,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:45:49,859 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:45:49,963 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:45:50,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:50,964 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:45:51,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:52,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:53,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:53,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:54,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:55,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:56,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:57,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:57,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:45:57,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:45:58,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:45:59,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:45:59,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:00,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:01,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:02,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:03,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:04,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:04,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:04,851 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:46:05,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:06,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:07,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:08,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:09,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:09,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:10,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:11,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:12,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:12,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:46:12,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:46:13,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:14,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:14,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:15,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:16,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:17,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:18,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:19,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:19,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:20,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:21,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:21,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:46:21,971 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:46:21,972 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:46:21,973 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:46:21,991 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:46:22,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:22,991 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:46:23,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:24,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:25,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:25,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:26,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:27,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:27,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:46:27,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:46:28,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:29,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:30,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:31,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:31,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:32,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:33,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:34,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:34,855 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:46:35,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:36,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:36,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:37,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:38,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:39,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:40,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:41,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:42,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:42,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:42,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:46:42,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:46:43,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:44,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:45,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:46,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:47,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:47,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:48,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:49,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:50,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:51,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:51,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:46:51,593 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:46:51,593 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:46:51,594 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:46:52,017 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:46:52,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:52,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:53,018 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:46:53,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:54,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:55,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:56,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:57,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:57,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:46:57,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:46:57,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:46:58,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:46:59,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:00,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:01,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:02,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:03,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:03,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:04,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:04,860 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:47:05,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:06,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:07,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:08,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:08,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:09,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:10,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:11,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:12,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:12,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:47:12,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:47:13,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:13,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:14,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:15,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:16,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:17,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:18,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:19,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:19,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:20,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:20,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:47:20,520 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:47:20,520 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:47:20,522 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:47:21,042 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:47:21,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:22,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:23,044 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:47:23,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:24,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:24,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:25,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:26,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:27,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:27,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:47:27,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:47:28,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:29,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:29,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:30,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:31,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:32,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:33,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:34,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:34,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:34,863 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:47:35,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:36,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:37,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:38,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:39,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:39,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:40,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:41,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:42,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:42,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:47:42,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:47:43,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:44,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:45,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:45,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:46,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:47,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:48,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:49,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:50,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:50,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:51,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:52,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:53,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:53,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:47:53,776 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:47:53,776 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:47:53,778 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:47:54,077 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:47:54,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:55,078 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:47:55,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:56,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:47:56,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:57,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:57,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:47:57,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:47:58,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:47:59,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:00,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:01,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:02,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:02,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:03,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:04,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:04,866 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:48:05,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:06,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:07,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:07,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:08,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:09,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:10,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:11,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:12,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:12,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:48:12,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:48:12,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:13,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:14,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:15,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:16,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:17,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:18,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:18,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:19,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:20,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:21,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:22,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:23,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:23,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:24,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:25,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:26,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:27,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:48:27,157 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:48:27,157 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:48:27,159 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:48:27,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:27,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:48:27,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:48:28,113 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:48:28,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:28,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:29,115 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:48:29,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:30,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:31,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:32,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:33,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:34,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:34,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:34,870 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:48:35,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:36,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:37,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:38,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:39,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:39,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:40,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:41,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:42,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:42,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:48:42,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:48:43,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:44,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:45,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:45,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:46,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:47,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:48,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:49,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:50,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:51,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:51,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:52,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:53,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:54,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:55,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:56,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:48:56,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:56,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:48:56,958 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:48:56,959 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:48:56,960 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:48:57,140 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:48:57,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:57,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:48:57,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:48:58,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:48:59,142 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:48:59,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:00,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:01,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:02,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:02,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:03,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:04,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:04,875 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:49:05,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:06,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:07,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:07,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:08,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:09,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:10,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:11,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:12,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:12,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:49:12,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:49:13,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:13,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:14,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:15,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:16,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:17,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:18,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:18,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:19,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:20,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:21,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:22,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:23,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:23,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:24,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:25,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:26,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:26,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:49:26,886 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:49:26,887 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:49:26,888 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:49:27,163 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:49:27,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:27,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:49:27,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:49:28,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:29,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:29,165 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:49:29,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:30,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:31,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:32,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:33,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:34,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:34,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:34,879 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:49:35,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:36,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:37,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:38,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:39,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:40,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:40,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:41,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:42,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:42,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:49:42,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:49:43,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:44,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:45,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:45,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:46,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:47,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:48,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:49,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:50,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:51,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:51,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:52,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:53,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:54,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:55,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:56,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:49:56,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:57,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:57,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:49:57,853 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:49:57,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:49:58,011 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:49:58,011 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:49:58,013 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:49:58,195 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:49:58,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:49:59,196 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:49:59,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:00,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:01,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:01,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:02,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:03,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:04,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:04,883 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:50:05,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:06,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:06,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:07,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:08,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:09,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:10,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:11,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:12,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:12,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:12,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:50:12,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:50:13,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:14,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:15,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:16,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:17,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:18,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:18,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:19,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:20,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:21,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:22,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:23,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:23,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:24,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:25,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:25,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:50:25,771 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:50:25,772 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:50:25,773 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:50:26,222 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:50:26,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:27,223 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:50:27,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:27,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:50:27,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:50:28,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:29,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:29,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:30,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:31,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:32,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:33,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:34,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:34,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:34,887 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:50:35,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:36,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:37,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:38,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:39,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:39,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:40,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:41,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:42,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:42,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:50:42,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:50:43,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:44,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:45,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:45,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:46,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:47,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:48,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:49,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:50,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:50,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:51,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:52,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:53,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:54,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:55,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:50:55,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:56,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:50:56,306 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:50:56,306 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:50:56,307 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:50:56,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:57,253 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:50:57,253 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:50:57,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:57,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:50:57,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:50:58,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:50:59,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:00,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:01,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:01,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:02,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:03,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:04,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:04,891 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:51:05,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:06,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:06,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:07,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:08,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:09,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:10,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:11,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:12,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:12,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:12,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:51:12,853 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:51:13,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:14,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:15,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:16,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:17,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:17,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:18,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:19,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:20,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:21,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:22,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:23,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:23,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:24,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:25,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:26,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:27,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:27,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:51:27,852 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:51:28,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:51:28,605 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:51:28,605 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:51:28,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:28,606 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:51:29,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:29,288 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:51:29,289 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:51:29,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:30,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:31,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:32,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:33,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:34,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:34,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:34,895 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:51:35,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:36,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:37,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:38,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:39,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:40,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:40,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:41,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:42,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:42,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:51:42,853 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:51:43,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:44,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:45,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:45,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:46,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:47,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:48,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:49,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:50,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:51,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:51,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:52,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:53,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:54,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:55,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:56,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:51:56,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:57,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:57,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:51:57,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:51:58,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:51:59,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:51:59,695 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:51:59,696 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:51:59,697 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:52:00,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:00,319 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:52:00,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:01,320 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:52:01,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:01,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:02,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:03,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:04,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:04,899 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:52:05,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:06,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:06,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:07,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:08,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:09,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:10,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:11,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:12,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:12,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:52:12,854 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:52:12,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:13,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:14,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:15,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:16,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:17,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:18,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:18,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:19,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:20,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:21,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:22,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:23,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:24,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:24,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:25,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:26,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:27,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:52:27,862 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:52:27,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:28,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:29,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:29,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:52:29,822 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:52:29,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:52:29,824 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:52:30,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:30,343 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:52:30,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:31,344 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:52:31,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:32,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:33,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:34,904 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:52:34,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:34,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:35,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:36,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:37,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:38,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:39,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:40,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:40,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:41,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:42,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:52:42,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:52:42,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:43,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:45,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:46,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:46,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:47,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:48,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:49,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:50,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:51,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:51,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:52,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:53,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:54,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:55,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:56,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:57,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:57,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:52:57,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:52:57,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:52:58,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:52:59,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:00,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:01,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:02,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:02,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:03,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:04,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:04,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:53:04,386 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:53:04,386 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:53:04,387 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:53:04,908 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:53:05,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:05,376 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:53:06,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:06,377 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:53:07,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:07,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:08,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:09,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:10,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:11,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:12,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:12,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:53:12,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:53:13,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:13,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:14,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:15,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:16,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:17,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:18,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:18,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:19,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:20,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:21,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:22,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:23,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:23,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:24,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:25,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:26,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:27,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:27,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:53:27,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:53:28,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:29,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:29,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:30,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:31,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:32,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:33,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:34,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:34,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:34,913 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:53:35,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:36,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:36,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:53:36,717 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:53:36,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:53:36,719 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:53:37,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:37,408 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:53:38,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:38,408 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:53:39,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:40,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:40,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:41,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:42,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:42,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:53:42,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:53:43,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:44,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:45,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:45,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:46,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:47,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:48,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:49,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:50,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:50,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:51,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:52,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:53,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:54,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:55,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:56,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:56,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:53:57,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:57,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:53:57,857 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:53:58,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:53:59,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:00,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:01,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:01,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:02,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:03,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:04,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:04,922 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:54:05,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:06,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:06,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:54:06,723 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:54:06,724 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:54:06,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:06,725 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:54:07,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:07,430 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:54:08,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:08,431 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:54:09,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:10,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:11,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:12,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:12,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:12,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:54:12,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:54:13,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:14,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:15,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:16,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:17,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:17,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:18,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:19,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:20,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:21,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:22,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:22,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:23,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:24,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:25,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:26,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:27,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:27,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:54:27,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:54:28,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:28,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:29,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:30,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:31,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:32,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:33,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:33,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:34,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:34,926 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:54:35,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:36,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:37,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:37,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:54:37,347 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:54:37,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:54:37,349 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:54:37,462 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:54:38,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:39,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:39,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:40,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:40,464 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:54:41,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:42,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:42,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:54:42,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:54:43,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:44,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:44,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:45,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:46,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:47,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:48,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:49,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:49,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:50,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:51,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:52,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:53,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:54,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:54,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:54:55,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:56,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:57,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:57,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:54:57,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:54:58,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:59,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:54:59,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:00,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:01,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:02,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:03,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:04,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:04,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:04,932 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:55:05,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:06,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:06,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:55:06,920 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:55:06,920 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:55:06,922 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:55:07,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:07,486 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:55:08,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:08,488 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:55:09,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:10,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:10,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:11,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:12,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:12,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:55:12,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:55:13,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:14,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:15,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:16,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:16,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:17,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:18,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:19,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:20,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:21,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:22,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:22,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:23,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:24,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:25,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:26,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:27,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:27,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:27,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:55:27,858 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:55:28,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:29,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:30,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:31,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:32,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:33,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:33,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:34,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:34,937 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:55:35,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:36,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:37,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:38,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:38,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:39,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:39,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:55:39,914 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:55:39,914 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:55:39,916 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:55:40,523 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:55:40,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:41,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:42,525 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:55:42,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:42,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:55:42,859 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:55:43,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:43,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:44,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:45,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:46,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:47,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:48,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:49,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:49,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:50,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:51,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:52,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:53,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:54,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:54,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:55,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:56,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:57,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:57,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:55:57,859 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:55:58,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:55:59,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:55:59,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:00,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:01,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:02,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:03,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:04,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:04,937 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:56:04,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:05,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:06,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:07,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:08,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:09,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:10,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:10,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:11,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:12,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:12,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:56:12,663 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:56:12,664 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:56:12,669 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:56:13,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:56:13,079 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:56:13,556 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:56:13,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:14,557 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:56:14,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:15,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:16,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:16,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:17,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:18,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:19,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:20,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:21,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:22,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:22,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:23,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:24,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:25,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:26,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:27,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:27,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:28,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:56:28,078 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:56:28,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:29,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:30,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:31,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:32,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:32,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:33,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:34,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:34,940 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:56:35,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:36,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:37,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:37,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:38,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:39,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:40,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:41,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:42,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:43,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:56:43,090 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:56:43,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:43,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:44,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:45,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:46,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:56:46,458 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:56:46,458 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:56:46,460 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:56:46,592 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:56:46,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:47,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:48,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:48,594 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:56:48,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:49,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:50,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:51,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:52,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:53,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:54,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:54,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:55,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:56,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:57,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:58,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:56:58,084 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:56:58,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:56:59,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:56:59,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:00,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:01,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:02,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:03,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:04,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:04,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:04,944 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:57:05,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:06,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:07,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:08,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:09,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:10,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:10,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:11,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:12,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:13,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:57:13,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:57:13,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:14,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:15,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:15,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:16,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:17,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:18,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:19,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:57:19,160 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:57:19,160 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:57:19,166 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:57:19,623 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:57:19,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:20,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:20,624 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:57:20,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:21,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:22,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:23,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:24,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:25,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:26,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:26,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:27,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:28,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:57:28,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:57:28,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:29,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:30,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:31,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:31,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:32,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:33,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:34,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:34,949 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:57:35,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:36,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:36,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:37,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:38,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:39,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:40,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:41,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:42,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:42,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:43,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:57:43,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:57:43,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:44,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:45,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:46,903 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:47,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:47,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:48,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:49,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:50,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:51,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:57:51,843 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:57:51,843 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:57:51,849 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:57:52,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:52,654 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:57:52,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:53,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:54,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:54,656 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:57:55,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:56,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:57,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:58,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:57:58,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:57:58,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:57:58,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:57:59,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:00,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:01,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:02,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:03,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:03,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:04,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:04,954 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:58:05,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:06,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:07,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:08,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:08,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:09,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:10,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:11,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:12,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:13,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:58:13,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:58:13,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:14,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:14,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:15,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:16,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:17,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:18,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:19,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:19,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:20,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:21,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:22,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:23,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:23,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:58:23,420 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:58:23,421 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:58:23,426 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:58:23,682 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:58:24,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:24,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:25,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:26,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:26,685 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:58:27,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:28,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:58:28,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:58:28,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:29,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:30,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:30,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:31,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:32,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:33,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:34,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:34,957 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:58:35,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:35,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:36,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:37,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:38,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:39,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:40,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:41,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:41,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:42,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:43,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:58:43,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:58:43,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:44,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:45,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:46,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:47,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:47,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:48,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:49,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:50,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:51,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:52,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:52,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:53,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:54,258 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:55,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:58:55,032 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:58:55,033 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:58:55,038 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:58:55,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:55,709 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:58:56,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:56,710 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:58:57,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:58,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:58:58,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:58:58,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:58:58,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:58:59,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:00,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:01,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:02,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:03,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:03,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:04,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:04,961 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:59:05,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:06,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:07,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:08,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:08,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:09,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:10,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:11,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:12,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:13,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:59:13,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:59:13,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:14,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:14,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:15,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:16,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:17,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:18,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:19,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:19,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:20,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:21,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:22,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:23,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:24,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:24,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:24,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:59:24,619 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:59:24,620 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:59:24,621 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:59:24,732 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:59:25,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:26,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:26,734 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:59:27,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:28,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:59:28,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:59:28,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:29,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:30,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:30,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:31,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:32,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:33,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:34,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:34,965 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 17:59:35,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:35,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:36,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:37,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:38,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:39,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:40,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:41,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:41,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:42,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:43,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:59:43,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:59:43,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:44,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:45,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:46,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:47,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:47,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:48,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:49,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:50,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:51,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:52,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:52,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:53,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:54,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:55,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:55,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 17:59:55,998 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 17:59:55,998 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 17:59:55,999 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 17:59:56,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:56,764 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 17:59:56,765 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 17:59:57,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:58,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 17:59:58,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 17:59:58,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 17:59:58,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 17:59:59,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:00,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:01,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:02,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:03,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:03,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:04,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:04,968 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:00:05,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:06,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:07,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:08,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:08,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:09,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:10,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:11,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:12,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:13,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:00:13,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:00:13,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:14,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:14,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:15,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:16,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:17,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:18,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:19,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:19,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:20,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:21,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:22,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:23,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:00:23,547 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:00:23,547 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:00:23,551 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:00:23,793 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:00:23,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:24,794 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:00:24,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:25,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:25,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:26,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:27,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:28,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:00:28,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:00:28,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:29,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:30,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:31,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:31,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:32,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:33,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:34,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:34,973 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:00:35,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:36,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:36,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:37,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:38,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:39,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:40,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:41,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:42,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:42,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:43,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:00:43,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:00:43,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:44,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:45,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:46,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:47,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:48,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:49,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:49,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:51,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:52,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:52,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:53,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:53,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:00:53,516 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:00:53,517 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:00:53,518 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:00:53,833 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:00:54,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:54,834 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:00:55,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:56,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:57,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:58,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:00:58,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:00:58,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:00:58,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:00:59,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:00,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:01,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:02,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:03,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:03,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:04,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:04,977 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:01:05,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:06,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:07,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:08,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:08,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:09,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:10,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:11,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:12,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:13,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:13,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:01:13,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:01:14,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:14,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:15,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:16,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:17,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:18,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:19,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:19,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:20,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:21,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:22,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:23,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:23,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:01:23,991 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:01:23,991 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:01:23,993 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:01:24,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:24,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:24,863 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:01:24,864 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:01:25,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:26,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:27,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:28,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:01:28,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:01:28,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:29,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:30,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:30,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:31,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:32,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:33,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:34,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:34,981 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:01:35,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:35,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:36,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:37,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:38,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:39,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:40,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:41,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:41,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:42,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:43,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:01:43,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:01:43,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:44,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:45,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:46,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:47,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:47,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:48,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:49,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:50,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:51,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:52,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:52,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:53,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:54,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:55,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:01:55,291 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:01:55,291 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:01:55,293 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:01:55,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:55,890 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:01:56,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:56,892 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:01:57,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:58,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:01:58,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:01:58,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:01:58,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:01:59,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:00,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:01,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:02,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:03,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:03,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:04,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:04,985 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:02:05,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:06,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:07,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:08,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:08,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:09,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:10,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:11,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:12,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:13,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:02:13,088 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:02:13,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:14,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:14,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:15,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:16,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:17,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:18,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:19,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:19,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:20,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:21,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:22,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:23,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:24,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:24,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:25,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:25,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:02:25,776 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:02:25,777 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:02:25,778 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:02:25,918 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:02:26,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:26,920 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:02:27,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:28,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:02:28,089 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:02:28,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:29,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:29,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:30,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:31,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:32,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:33,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:34,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:34,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:34,989 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:02:35,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:36,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:37,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:38,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:39,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:39,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:40,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:41,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:42,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:43,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:02:43,089 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:02:43,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:44,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:44,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:45,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:46,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:47,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:48,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:49,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:50,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:50,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:51,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:52,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:53,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:54,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:55,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:55,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:02:56,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:56,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:02:56,916 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:02:56,917 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:02:56,918 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:02:56,950 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:02:57,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:58,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:02:58,089 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:02:58,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:02:58,952 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:02:59,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:00,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:01,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:01,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:02,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:03,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:04,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:04,993 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:03:05,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:06,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:06,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:07,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:08,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:09,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:10,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:11,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:12,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:12,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:13,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:03:13,089 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:03:13,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:14,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:15,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:16,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:17,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:17,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:18,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:19,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:20,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:21,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:22,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:23,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:23,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:24,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:25,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:26,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:27,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:28,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:03:28,090 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:03:28,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:28,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:03:28,717 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:03:28,717 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:03:28,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:28,719 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:03:28,990 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:03:29,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:30,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:30,992 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:03:31,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:32,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:33,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:34,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:34,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:34,998 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:03:35,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:36,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:37,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:38,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:39,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:40,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:40,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:41,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:42,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:43,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:03:43,090 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:03:43,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:44,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:45,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:45,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:46,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:47,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:48,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:49,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:50,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:51,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:51,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:52,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:53,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:54,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:55,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:56,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:56,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:03:57,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:58,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:03:58,090 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:03:58,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:03:58,247 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:03:58,247 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:03:58,249 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:03:58,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:03:59,025 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:03:59,026 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:03:59,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:00,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:01,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:02,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:02,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:03,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:04,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:05,002 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:04:05,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:06,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:07,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:08,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:08,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:09,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:10,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:11,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:12,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:13,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:04:13,091 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:04:13,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:13,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:14,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:15,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:16,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:17,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:18,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:18,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:19,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:20,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:21,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:22,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:23,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:23,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:24,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:25,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:26,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:27,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:28,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:04:28,098 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:04:28,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:29,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:29,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:04:29,306 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:04:29,307 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:04:29,308 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:04:29,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:30,055 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:04:30,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:31,058 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:04:31,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:32,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:33,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:34,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:34,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:35,006 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:04:35,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:36,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:37,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:38,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:39,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:40,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:40,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:41,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:42,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:43,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:04:43,092 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:04:43,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:44,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:45,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:46,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:46,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:47,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:48,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:49,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:50,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:51,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:51,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:52,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:53,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:54,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:55,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:56,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:04:56,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:57,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:57,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:04:57,993 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:04:57,994 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:04:57,995 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:04:58,087 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:04:58,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:04:58,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:04:58,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:04:59,089 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:04:59,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:00,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:01,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:01,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:02,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:03,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:04,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:05,010 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:05:05,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:06,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:07,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:07,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:08,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:09,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:10,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:11,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:12,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:12,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:13,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:05:13,147 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:05:13,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:14,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:15,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:16,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:17,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:18,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:18,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:19,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:20,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:21,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:22,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:23,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:23,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:24,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:25,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:26,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:27,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:28,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:05:28,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:05:28,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:29,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:29,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:05:29,656 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:05:29,656 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:05:29,658 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:05:30,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:30,120 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:05:31,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:31,122 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:05:32,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:33,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:34,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:34,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:35,014 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:05:35,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:36,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:37,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:38,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:39,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:40,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:40,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:41,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:42,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:43,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:43,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:05:43,150 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:05:44,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:45,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:45,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:46,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:47,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:48,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:49,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:50,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:51,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:51,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:52,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:53,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:54,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:55,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:56,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:56,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:05:57,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:58,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:58,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:05:58,150 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:05:59,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:05:59,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:05:59,985 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:05:59,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:05:59,987 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:06:00,154 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:06:00,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:01,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:01,155 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:06:02,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:02,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:03,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:04,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:05,019 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:06:05,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:06,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:07,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:08,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:08,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:09,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:10,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:11,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:12,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:13,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:13,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:06:13,150 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:06:13,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:14,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:15,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:16,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:17,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:18,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:18,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:19,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:20,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:21,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:22,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:23,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:23,822 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:24,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:25,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:26,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:27,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:28,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:28,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:06:28,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:06:29,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:29,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:06:29,297 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:06:29,298 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:06:29,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:29,299 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:06:30,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:30,182 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:06:31,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:31,183 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:06:32,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:33,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:34,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:34,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:35,022 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:06:35,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:36,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:37,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:38,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:39,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:40,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:40,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:41,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:42,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:43,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:43,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:06:43,150 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:06:44,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:45,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:45,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:46,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:47,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:48,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:49,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:50,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:51,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:51,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:52,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:53,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:54,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:55,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:56,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:56,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:06:57,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:58,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:06:58,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:06:58,154 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:06:59,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:00,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:00,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:07:00,562 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:07:00,562 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:07:00,564 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:07:01,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:01,207 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:07:01,207 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:07:02,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:02,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:03,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:04,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:05,026 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:07:05,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:06,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:07,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:08,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:08,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:09,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:10,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:11,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:12,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:13,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:13,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:07:13,155 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:07:13,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:14,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:15,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:16,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:17,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:18,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:18,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:19,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:20,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:21,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:22,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:23,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:24,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:24,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:25,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:26,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:27,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:28,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:28,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:07:28,158 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:07:29,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:29,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:30,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:30,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:07:30,505 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:07:30,506 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:07:30,507 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:07:31,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:31,231 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:07:31,231 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:07:32,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:33,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:34,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:34,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:35,031 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:07:35,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:36,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:37,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:38,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:39,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:39,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:40,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:41,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:42,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:43,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:07:43,156 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:07:43,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:44,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:45,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:45,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:46,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:47,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:48,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:49,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:50,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:51,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:51,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:52,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:53,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:54,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:55,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:56,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:56,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:07:57,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:58,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:07:58,163 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:07:58,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:59,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:07:59,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:07:59,333 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:07:59,333 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:07:59,334 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:08:00,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:00,264 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:08:01,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:01,265 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:08:02,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:02,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:03,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:04,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:05,035 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:08:05,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:06,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:07,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:08,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:08,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:09,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:10,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:11,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:12,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:13,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:08:13,156 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:08:13,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:13,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:14,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:15,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:16,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:17,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:18,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:18,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:19,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:20,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:21,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:22,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:23,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:23,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:24,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:25,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:26,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:27,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:28,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:08:28,156 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:08:28,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:29,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:29,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:29,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:08:29,488 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:08:29,489 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:08:29,490 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:08:30,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:30,291 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:08:31,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:31,293 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:08:32,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:33,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:34,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:34,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:35,040 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:08:35,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:36,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:37,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:38,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:39,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:40,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:40,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:41,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:42,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:43,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:08:43,156 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:08:43,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:44,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:45,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:45,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:46,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:47,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:48,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:49,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:50,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:51,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:51,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:52,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:53,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:54,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:55,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:56,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:56,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:08:57,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:58,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:08:58,160 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:08:58,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:59,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:08:59,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:08:59,886 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:08:59,886 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:08:59,887 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:09:00,317 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:09:00,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:01,319 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:09:01,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:02,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:02,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:03,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:04,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:05,043 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:09:05,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:06,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:07,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:08,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:08,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:09,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:10,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:11,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:12,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:13,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:09:13,160 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:09:13,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:13,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:14,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:15,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:16,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:17,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:18,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:19,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:19,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:20,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:21,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:22,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:23,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:24,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:24,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:25,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:26,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:27,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:28,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:09:28,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:09:28,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:29,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:29,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:29,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:09:29,615 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:09:29,615 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:09:29,616 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:09:30,350 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:09:30,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:31,352 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:09:31,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:32,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:33,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:34,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:35,047 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:09:35,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:35,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:36,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:37,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:38,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:39,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:40,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:40,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:41,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:42,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:43,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:09:43,167 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:09:43,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:44,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:45,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:45,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:46,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:47,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:48,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:49,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:50,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:51,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:51,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:52,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:53,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:54,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:55,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:56,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:09:56,414 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:57,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:58,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:09:58,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:09:58,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:09:59,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:00,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:01,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:01,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:01,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:10:01,648 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:10:01,648 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:10:01,650 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:10:02,382 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:10:02,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:03,384 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:10:03,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:04,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:05,051 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:10:05,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:06,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:07,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:07,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:08,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:09,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:10,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:11,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:12,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:13,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:13,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:10:13,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:10:13,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:14,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:15,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:16,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:17,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:18,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:19,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:19,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:20,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:21,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:22,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:23,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:24,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:24,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:25,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:26,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:27,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:28,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:10:28,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:10:28,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:29,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:29,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:30,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:31,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:32,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:33,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:34,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:35,055 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:10:35,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:35,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:10:35,297 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:10:35,298 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:10:35,299 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:10:35,420 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:10:35,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:36,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:37,422 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:10:37,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:38,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:39,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:40,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:40,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:41,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:42,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:43,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:10:43,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:10:43,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:44,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:45,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:46,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:46,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:47,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:48,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:49,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:50,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:51,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:51,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:52,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:53,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:54,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:55,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:56,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:57,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:10:57,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:58,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:10:58,172 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:10:58,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:10:59,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:00,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:01,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:02,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:02,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:11:02,944 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:11:02,944 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:11:02,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:02,946 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:11:03,445 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:11:03,446 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:11:03,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:04,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:05,059 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:11:05,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:06,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:07,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:08,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:08,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:09,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:10,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:11,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:12,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:13,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:11:13,172 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:11:13,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:13,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:14,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:15,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:16,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:17,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:18,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:19,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:19,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:20,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:21,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:22,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:23,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:24,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:24,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:25,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:26,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:27,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:28,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:11:28,172 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:11:28,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:29,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:29,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:30,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:31,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:32,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:33,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:34,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:35,062 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:11:35,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:35,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:35,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:11:35,905 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:11:35,905 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:11:35,907 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:11:36,474 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:11:36,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:37,475 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:11:37,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:38,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:39,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:40,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:40,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:41,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:42,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:43,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:11:43,172 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:11:43,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:44,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:45,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:46,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:46,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:47,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:48,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:49,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:50,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:51,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:51,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:52,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:53,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:54,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:55,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:56,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:57,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:11:57,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:58,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:11:58,172 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:11:58,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:11:59,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:00,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:01,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:02,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:02,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:03,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:04,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:04,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:12:04,783 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:12:04,784 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:12:04,785 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:12:05,066 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:12:05,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:05,504 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:12:05,505 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:12:06,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:07,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:07,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:08,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:09,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:10,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:11,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:12,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:13,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:12:13,173 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:12:13,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:13,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:14,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:15,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:16,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:17,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:18,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:19,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:19,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:20,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:21,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:22,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:23,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:24,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:25,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:25,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:26,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:27,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:28,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:12:28,174 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:12:28,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:29,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:30,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:30,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:31,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:32,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:33,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:12:33,554 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:12:33,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:12:33,556 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:12:33,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:34,529 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:12:34,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:35,070 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:12:35,530 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:12:35,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:36,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:36,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:37,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:38,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:39,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:40,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:41,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:41,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:42,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:43,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:12:43,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:12:43,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:44,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:45,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:46,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:46,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:47,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:48,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:49,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:50,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:51,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:52,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:52,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:53,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:54,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:55,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:56,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:57,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:58,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:12:58,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:12:58,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:12:58,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:12:59,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:00,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:01,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:02,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:03,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:04,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:04,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:05,073 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:13:05,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:13:05,701 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:13:05,702 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:13:05,704 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:13:06,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:06,558 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:13:06,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:07,559 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:13:07,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:08,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:09,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:09,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:10,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:11,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:12,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:13,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:13:13,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:13:13,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:14,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:15,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:15,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:16,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:17,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:18,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:19,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:20,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:20,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:21,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:22,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:23,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:24,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:25,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:26,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:26,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:27,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:28,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:13:28,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:13:28,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:29,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:30,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:31,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:32,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:32,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:33,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:34,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:35,078 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:13:35,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:36,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:37,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:37,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:38,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:13:38,720 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:13:38,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:13:38,722 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:13:39,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:39,588 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:13:39,588 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:13:40,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:41,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:42,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:43,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:43,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:13:43,175 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:13:43,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:44,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:45,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:46,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:47,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:48,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:49,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:49,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:50,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:51,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:52,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:53,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:54,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:55,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:55,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:13:56,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:57,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:58,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:13:58,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:13:58,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:13:59,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:00,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:01,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:01,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:02,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:03,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:04,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:05,083 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:14:05,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:06,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:06,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:07,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:08,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:08,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:14:08,783 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:14:08,783 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:14:08,784 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:14:09,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:09,616 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:14:09,617 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:14:10,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:11,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:12,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:12,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:13,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:14:13,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:14:13,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:14,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:15,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:16,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:17,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:17,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:18,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:19,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:20,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:21,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:22,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:23,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:23,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:24,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:25,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:26,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:27,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:28,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:14:28,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:14:28,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:28,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:29,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:30,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:31,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:32,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:33,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:34,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:34,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:35,087 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:14:35,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:36,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:37,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:38,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:39,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:39,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:39,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:14:39,449 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:14:39,450 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:14:39,451 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:14:39,645 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:14:40,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:41,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:41,647 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:14:42,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:43,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:14:43,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:14:43,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:44,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:44,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:45,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:46,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:47,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:48,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:49,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:50,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:50,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:51,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:52,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:53,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:54,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:55,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:56,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:56,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:14:57,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:58,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:14:58,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:14:58,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:14:59,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:00,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:01,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:02,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:02,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:03,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:04,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:05,095 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:15:05,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:06,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:07,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:07,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:08,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:09,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:09,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:15:09,317 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:15:09,317 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:15:09,319 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:15:09,672 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:15:09,673 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:15:10,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:11,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:12,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:13,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:15:13,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:15:13,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:13,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:14,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:15,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:16,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:17,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:18,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:19,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:19,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:20,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:21,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:22,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:23,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:24,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:24,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:25,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:26,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:27,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:28,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:15:28,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:15:28,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:29,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:29,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:30,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:31,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:32,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:33,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:34,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:35,093 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:15:35,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:35,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:36,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:37,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:38,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:39,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:40,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:40,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:41,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:41,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:15:41,624 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:15:41,624 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:15:41,626 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:15:41,710 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:15:42,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:43,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:15:43,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:15:43,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:43,713 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:15:44,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:45,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:46,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:46,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:47,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:48,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:49,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:50,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:51,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:52,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:52,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:53,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:54,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:55,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:56,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:57,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:15:57,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:58,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:15:58,176 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:15:58,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:15:59,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:00,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:01,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:02,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:03,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:03,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:04,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:05,102 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:16:05,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:06,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:07,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:08,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:08,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:09,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:10,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:10,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:16:10,599 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:16:10,600 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:16:10,601 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:16:10,744 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:16:11,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:11,746 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:16:12,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:13,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:16:13,177 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:16:13,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:13,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:14,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:15,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:16,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:17,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:18,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:19,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:19,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:20,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:21,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:22,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:23,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:24,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:25,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:25,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:26,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:27,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:28,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:16:28,177 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:16:28,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:29,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:30,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:31,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:31,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:32,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:33,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:34,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:35,104 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:16:35,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:36,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:37,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:37,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:38,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:39,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:40,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:41,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:42,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:42,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:42,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:16:42,924 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:16:42,924 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:16:42,929 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:16:43,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:16:43,344 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:16:43,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:43,777 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:16:44,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:44,779 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:16:45,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:46,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:47,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:48,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:48,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:49,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:50,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:51,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:52,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:53,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:54,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:54,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:55,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:56,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:57,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:58,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:16:58,343 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:16:58,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:16:59,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:16:59,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:00,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:01,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:02,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:03,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:04,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:05,108 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:17:05,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:05,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:06,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:07,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:08,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:09,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:10,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:10,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:11,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:12,830 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:13,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:17:13,345 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:17:13,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:17:13,730 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:17:13,731 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:17:13,736 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:17:13,812 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:17:14,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:14,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:15,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:16,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:16,813 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:17:16,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:17,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:18,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:19,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:20,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:21,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:22,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:22,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:23,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:24,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:25,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:26,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:27,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:27,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:28,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:17:28,344 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:17:28,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:29,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:30,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:31,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:32,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:32,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:33,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:34,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:35,112 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:17:35,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:36,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:37,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:38,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:38,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:39,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:40,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:41,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:42,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:43,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:43,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:17:43,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:17:43,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:44,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:45,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:17:45,510 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:17:45,510 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:17:45,516 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:17:45,839 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:17:45,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:46,840 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:17:46,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:47,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:48,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:48,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:49,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:50,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:51,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:52,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:53,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:54,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:54,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:55,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:56,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:57,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:58,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:17:58,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:17:58,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:17:59,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:17:59,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:00,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:01,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:02,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:03,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:04,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:05,116 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:18:05,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:05,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:06,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:07,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:08,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:09,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:10,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:10,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:11,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:12,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:13,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:18:13,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:18:13,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:14,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:15,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:16,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:16,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:17,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:18,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:19,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:18:19,779 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:18:19,780 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:18:19,783 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:18:19,868 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:18:20,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:21,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:21,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:22,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:22,870 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:18:23,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:24,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:25,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:26,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:27,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:27,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:28,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:28,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:18:28,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:18:29,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:30,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:31,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:32,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:33,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:33,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:34,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:35,120 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:18:35,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:36,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:37,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:38,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:39,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:39,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:40,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:41,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:42,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:43,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:43,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:18:43,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:18:44,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:44,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:45,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:46,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:47,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:48,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:49,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:50,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:50,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:50,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:18:50,385 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:18:50,386 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:18:50,391 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:18:50,904 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:18:51,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:52,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:52,905 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:18:53,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:54,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:55,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:55,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:18:56,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:57,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:58,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:18:58,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:18:58,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:18:59,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:00,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:01,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:01,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:02,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:03,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:04,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:05,124 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:19:05,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:06,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:06,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:07,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:08,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:09,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:10,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:11,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:11,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:12,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:13,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:13,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:19:13,346 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:19:14,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:15,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:16,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:16,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:17,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:18,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:19,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:20,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:21,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:21,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:22,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:22,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:19:22,343 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:19:22,344 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:19:22,346 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:19:22,928 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:19:23,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:24,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:24,929 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:19:25,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:26,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:26,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:27,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:28,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:28,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:19:28,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:19:29,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:30,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:31,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:32,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:32,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:33,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:34,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:35,130 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:19:35,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:36,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:37,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:37,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:38,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:39,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:40,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:41,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:42,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:43,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:43,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:43,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:19:43,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:19:44,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:45,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:46,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:47,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:48,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:48,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:49,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:50,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:51,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:51,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:19:51,493 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:19:51,493 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:19:51,495 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:19:51,960 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:19:52,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:52,961 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:19:53,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:53,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:19:54,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:55,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:56,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:57,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:58,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:58,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:19:58,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:19:59,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:19:59,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:00,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:01,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:02,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:03,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:04,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:05,133 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:20:05,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:05,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:06,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:07,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:08,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:09,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:10,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:10,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:11,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:12,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:13,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:20:13,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:20:13,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:14,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:15,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:16,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:16,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:17,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:18,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:19,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:20,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:21,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:21,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:21,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:20:21,918 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:20:21,918 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:20:21,919 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:20:21,987 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:20:22,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:23,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:24,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:24,989 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:20:25,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:26,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:27,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:27,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:28,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:20:28,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:20:28,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:29,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:30,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:31,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:32,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:32,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:33,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:34,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:35,137 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:20:35,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:36,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:37,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:38,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:38,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:39,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:40,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:41,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:42,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:43,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:20:43,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:20:43,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:43,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:44,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:45,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:46,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:47,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:48,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:49,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:49,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:50,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:51,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:51,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:20:51,667 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:20:51,668 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:20:51,670 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:20:52,010 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:20:52,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:53,011 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:20:53,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:54,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:54,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:20:55,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:56,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:57,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:58,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:20:58,348 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:20:58,388 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:20:59,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:00,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:00,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:01,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:02,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:03,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:04,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:05,141 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:21:05,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:05,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:06,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:07,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:08,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:09,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:10,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:10,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:11,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:12,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:13,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:21:13,358 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:21:13,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:14,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:15,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:15,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:16,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:17,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:18,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:19,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:21:19,184 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:21:19,184 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:21:19,186 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:21:19,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:20,033 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:21:20,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:20,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:21,034 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:21:21,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:22,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:23,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:24,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:25,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:25,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:26,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:27,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:28,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:21:28,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:21:28,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:29,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:30,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:31,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:31,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:32,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:33,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:34,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:35,145 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:21:35,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:36,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:36,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:37,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:38,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:39,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:40,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:41,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:41,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:42,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:43,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:21:43,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:21:43,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:44,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:45,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:46,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:46,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:47,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:48,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:49,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:50,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:51,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:51,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:51,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:21:51,632 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:21:51,633 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:21:51,635 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:21:52,064 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:21:52,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:53,064 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:21:53,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:54,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:55,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:56,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:21:56,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:57,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:58,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:21:58,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:21:58,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:21:59,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:00,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:01,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:02,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:02,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:03,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:04,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:05,150 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:22:05,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:06,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:07,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:07,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:08,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:09,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:10,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:11,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:12,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:13,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:13,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:22:13,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:22:13,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:14,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:15,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:16,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:17,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:18,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:18,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:19,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:20,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:21,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:22,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:22,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:22:22,653 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:22:22,653 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:22:22,655 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:22:23,090 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:22:23,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:23,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:24,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:25,091 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:22:25,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:26,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:27,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:28,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:22:28,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:22:28,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:29,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:29,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:30,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:31,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:32,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:33,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:34,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:35,152 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:22:35,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:35,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:36,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:37,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:38,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:39,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:40,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:40,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:41,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:42,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:43,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:22:43,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:22:43,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:44,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:45,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:46,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:46,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:47,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:48,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:49,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:50,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:51,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:51,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:52,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:53,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:54,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:54,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:22:54,992 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:22:54,992 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:22:54,994 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:22:55,121 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:22:55,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:56,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:22:56,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:57,123 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:22:57,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:58,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:22:58,350 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:22:58,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:22:59,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:00,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:01,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:02,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:02,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:03,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:04,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:05,156 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:23:05,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:06,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:07,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:08,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:08,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:09,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:10,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:11,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:12,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:13,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:23:13,350 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:23:13,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:13,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:14,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:15,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:16,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:17,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:18,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:19,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:19,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:20,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:21,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:22,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:23,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:24,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:24,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:24,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:23:24,921 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:23:24,922 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:23:24,924 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:23:25,150 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:23:25,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:26,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:27,152 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:23:27,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:28,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:23:28,350 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:23:28,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:29,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:29,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:30,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:31,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:32,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:33,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:34,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:35,160 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:23:35,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:35,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:36,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:37,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:38,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:39,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:40,414 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:40,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:41,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:42,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:43,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:23:43,350 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:23:43,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:44,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:45,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:46,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:46,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:47,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:48,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:49,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:50,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:51,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:51,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:52,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:53,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:54,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:54,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:23:54,858 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:23:54,859 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:23:54,860 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:23:55,185 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:23:55,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:56,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:57,187 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:23:57,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:23:57,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:58,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:23:58,350 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:23:58,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:23:59,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:00,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:01,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:02,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:03,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:03,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:04,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:05,163 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:24:05,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:06,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:07,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:08,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:08,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:09,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:10,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:11,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:12,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:13,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:24:13,350 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:24:13,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:14,414 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:14,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:15,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:16,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:17,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:19,202 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:24:19,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:20,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:24,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:28,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:29,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:30,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:35,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:24:35,122 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:24:35,253 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:24:36,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:36,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:37,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:41,565 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:44,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:24:44,999 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:24:45,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:47,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:52,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:24:52,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:54,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:24:57,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:01,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:03,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:03,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:25:03,498 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:25:04,388 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:05,172 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:25:09,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:14,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:15,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:25:15,640 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:25:15,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:16,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:17,523 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:25:17,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:18,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:19,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:20,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:20,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:21,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:22,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:23,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:24,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:25,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:25,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:26,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:27,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:28,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:29,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:30,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:30,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:25:30,639 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:25:30,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:31,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:32,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:33,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:34,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:35,176 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:25:35,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:36,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:36,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:37,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:38,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:39,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:40,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:41,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:41,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:42,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:43,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:44,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:45,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:25:45,640 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:25:45,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:46,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:46,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:47,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:48,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:49,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:50,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:51,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:52,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:52,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:53,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:54,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:55,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:56,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:57,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:58,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:25:58,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:25:59,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:00,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:26:00,640 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:26:00,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:01,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:02,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:03,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:03,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:04,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:05,184 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:26:05,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:06,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:07,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:09,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:14,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:19,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:24,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:27,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:26:27,431 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:26:27,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:28,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:29,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:29,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:30,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:31,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:32,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:33,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:34,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:35,184 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:26:35,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:35,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:36,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:37,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:38,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:39,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:40,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:40,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:41,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:42,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:42,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:26:42,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:26:43,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:44,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:45,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:45,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:46,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:47,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:48,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:49,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:50,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:51,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:51,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:52,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:53,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:54,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:55,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:56,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:57,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:26:57,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:57,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:26:57,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:26:58,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:26:59,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:00,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:01,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:02,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:03,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:03,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:04,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:05,188 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:27:05,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:06,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:07,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:08,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:09,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:09,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:10,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:11,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:12,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:27:12,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:27:12,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:13,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:14,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:14,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:15,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:16,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:17,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:18,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:19,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:20,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:20,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:21,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:22,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:23,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:24,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:25,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:25,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:26,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:27,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:27:27,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:27:27,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:28,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:29,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:30,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:30,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:31,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:32,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:33,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:34,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:35,194 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:27:35,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:36,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:36,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:37,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:38,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:39,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:40,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:41,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:41,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:42,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:27:42,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:27:42,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:43,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:44,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:45,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:46,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:46,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:47,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:48,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:49,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:50,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:51,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:52,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:52,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:53,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:54,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:55,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:56,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:57,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:27:57,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:27:57,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:57,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:27:58,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:27:59,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:00,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:01,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:02,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:03,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:03,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:04,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:05,197 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:28:05,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:06,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:07,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:08,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:08,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:09,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:10,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:11,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:12,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:28:12,430 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:28:12,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:13,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:13,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:14,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:15,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:18,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:23,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:28,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:33,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:35,201 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:28:39,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:44,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:49,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:54,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:55,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:28:55,402 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:28:55,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:56,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:57,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:58,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:28:59,190 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:28:59,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:28:59,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:00,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:01,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:02,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:03,192 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:29:03,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:04,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:04,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:05,205 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:29:05,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:06,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:07,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:08,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:09,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:10,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:10,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:29:10,400 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:29:10,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:11,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:12,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:13,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:14,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:15,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:15,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:16,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:17,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:18,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:19,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:20,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:21,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:21,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:22,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:23,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:24,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:25,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:29:25,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:29:25,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:26,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:26,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:27,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:28,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:29,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:30,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:31,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:29:31,104 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:29:31,105 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:29:31,106 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:29:31,217 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:29:31,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:32,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:32,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:33,219 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:29:33,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:34,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:35,209 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:29:35,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:36,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:37,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:37,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:38,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:39,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:40,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:29:40,405 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:29:40,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:41,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:42,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:42,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:43,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:44,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:45,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:46,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:47,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:48,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:48,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:49,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:50,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:51,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:52,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:53,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:53,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:54,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:55,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:29:55,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:29:55,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:56,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:57,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:58,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:29:58,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:29:59,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:00,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:30:00,267 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:30:00,267 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:30:00,269 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:30:00,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:01,244 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:30:01,244 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:30:01,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:02,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:03,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:03,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:04,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:05,214 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:30:05,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:06,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:07,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:08,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:09,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:09,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:10,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:30:10,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:30:10,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:11,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:12,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:13,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:14,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:14,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:15,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:16,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:17,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:18,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:19,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:20,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:20,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:21,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:22,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:23,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:24,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:25,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:30:25,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:30:25,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:25,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:26,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:27,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:28,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:29,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:30,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:31,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:31,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:32,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:30:32,030 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:30:32,031 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:30:32,033 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:30:32,276 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:30:32,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:33,277 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:30:33,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:34,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:35,219 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:30:35,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:36,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:37,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:37,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:38,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:39,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:40,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:30:40,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:30:40,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:41,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:42,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:42,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:43,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:44,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:45,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:46,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:47,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:48,423 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:48,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:49,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:50,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:51,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:52,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:53,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:54,423 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:54,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:55,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:30:55,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:30:55,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:56,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:57,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:58,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:30:59,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:30:59,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:00,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:01,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:01,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:31:01,818 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:31:01,818 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:31:01,820 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:31:02,308 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:31:02,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:03,309 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:31:03,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:04,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:04,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:05,222 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:31:05,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:06,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:07,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:08,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:09,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:10,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:31:10,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:31:10,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:10,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:11,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:12,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:13,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:14,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:15,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:16,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:16,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:17,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:18,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:19,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:20,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:21,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:22,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:22,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:23,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:24,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:25,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:31:25,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:31:25,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:26,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:27,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:28,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:28,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:29,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:30,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:31,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:32,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:31:32,393 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:31:32,394 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:31:32,396 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:31:32,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:33,338 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:31:33,338 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:31:33,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:33,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:34,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:35,226 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:31:35,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:36,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:37,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:38,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:39,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:39,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:40,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:31:40,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:31:40,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:41,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:42,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:43,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:44,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:44,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:45,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:46,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:47,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:48,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:49,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:50,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:50,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:51,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:52,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:53,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:54,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:55,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:31:55,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:31:55,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:31:55,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:56,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:57,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:58,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:31:59,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:00,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:01,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:01,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:02,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:03,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:04,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:04,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:32:04,971 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:32:04,972 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:32:04,973 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:32:05,229 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:32:05,370 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:32:05,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:06,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:07,371 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:32:07,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:07,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:08,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:09,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:10,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:32:10,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:32:10,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:11,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:12,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:12,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:13,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:14,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:15,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:16,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:17,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:18,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:18,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:19,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:20,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:21,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:22,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:23,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:23,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:24,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:25,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:32:25,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:32:25,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:26,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:27,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:28,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:28,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:29,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:30,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:31,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:32,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:33,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:34,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:34,836 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:35,234 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:32:35,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:36,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:37,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:38,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:32:38,100 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:32:38,100 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:32:38,102 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:32:38,402 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:32:38,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:39,403 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:32:39,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:40,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:40,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:32:40,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:32:40,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:41,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:42,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:43,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:44,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:45,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:45,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:46,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:47,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:48,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:49,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:50,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:51,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:51,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:52,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:53,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:54,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:55,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:32:55,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:32:55,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:56,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:32:56,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:57,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:58,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:32:59,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:00,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:01,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:02,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:02,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:03,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:04,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:05,238 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:33:05,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:06,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:07,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:07,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:08,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:09,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:10,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:33:10,439 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:33:10,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:11,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:12,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:12,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:12,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:33:12,941 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:33:12,941 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:33:12,942 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:33:13,448 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:33:13,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:14,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:15,449 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:33:15,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:16,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:17,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:18,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:18,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:19,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:20,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:21,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:22,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:23,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:24,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:24,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:25,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:33:25,455 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:33:25,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:26,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:27,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:28,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:29,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:29,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:30,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:31,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:32,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:33,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:34,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:35,243 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:33:35,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:35,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:36,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:37,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:38,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:39,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:40,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:33:40,463 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:33:40,531 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:40,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:41,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:42,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:43,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:33:43,008 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:33:43,008 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:33:43,009 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:33:43,482 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:33:43,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:44,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:45,484 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:33:45,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:46,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:46,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:47,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:48,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:49,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:50,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:51,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:52,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:52,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:53,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:54,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:55,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:33:55,456 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:33:55,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:56,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:57,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:33:57,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:58,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:33:59,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:00,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:01,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:02,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:03,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:03,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:04,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:05,246 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:34:05,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:06,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:07,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:08,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:08,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:09,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:10,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:34:10,461 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:34:10,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:11,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:34:11,573 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:34:11,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:34:11,575 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:34:12,509 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:34:12,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:13,510 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:34:13,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:13,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:14,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:15,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:16,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:17,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:18,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:19,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:19,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:20,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:21,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:22,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:23,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:24,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:25,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:34:25,458 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:34:25,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:25,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:26,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:27,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:28,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:29,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:30,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:31,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:31,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:32,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:33,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:34,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:35,250 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:34:35,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:36,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:37,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:37,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:38,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:39,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:40,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:34:40,470 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:34:40,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:41,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:42,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:42,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:42,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:34:42,908 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:34:42,908 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:34:42,909 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:34:43,540 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:34:43,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:44,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:45,542 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:34:45,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:46,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:47,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:48,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:48,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:49,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:50,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:51,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:52,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:53,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:53,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:54,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:55,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:34:55,462 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:34:55,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:56,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:57,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:58,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:34:58,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:34:59,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:00,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:01,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:02,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:03,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:04,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:04,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:05,254 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:35:05,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:06,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:07,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:08,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:09,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:09,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:10,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:35:10,464 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:35:10,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:11,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:12,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:35:12,549 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:35:12,550 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:35:12,551 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:35:12,562 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:35:12,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:13,563 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:35:13,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:14,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:14,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:15,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:16,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:17,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:18,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:19,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:20,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:20,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:21,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:22,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:23,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:24,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:25,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:35:25,465 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:35:25,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:25,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:26,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:27,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:28,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:29,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:30,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:31,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:31,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:32,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:33,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:34,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:35,259 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:35:35,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:36,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:37,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:37,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:38,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:39,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:40,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:35:40,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:35:40,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:41,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:42,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:42,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:43,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:35:43,629 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:35:43,629 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:35:43,631 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:35:44,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:44,596 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:35:44,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:45,597 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:35:45,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:46,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:47,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:48,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:48,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:49,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:50,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:51,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:52,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:53,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:54,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:54,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:55,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:35:55,468 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:35:55,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:56,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:57,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:58,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:35:59,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:35:59,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:00,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:01,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:02,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:03,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:04,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:05,263 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:36:05,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:05,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:06,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:07,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:08,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:09,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:10,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:36:10,481 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:36:10,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:10,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:11,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:12,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:13,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:14,789 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:15,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:16,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:36:16,324 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:36:16,324 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:36:16,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:16,325 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:36:16,629 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:36:16,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:17,630 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:36:17,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:18,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:19,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:20,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:21,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:21,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:22,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:23,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:24,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:25,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:36:25,470 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:36:25,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:26,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:26,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:27,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:28,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:29,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:30,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:31,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:32,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:32,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:33,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:34,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:35,267 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:36:35,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:36,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:37,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:38,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:38,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:39,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:40,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:36:40,470 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:36:40,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:41,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:42,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:43,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:43,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:44,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:45,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:46,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:47,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:36:47,695 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:36:47,695 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:36:47,697 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:36:48,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:48,654 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:36:48,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:49,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:49,655 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:36:50,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:51,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:52,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:53,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:54,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:54,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:36:55,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:55,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:36:55,470 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:36:56,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:57,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:58,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:59,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:36:59,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:00,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:01,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:02,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:03,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:04,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:05,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:05,271 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:37:05,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:06,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:07,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:08,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:09,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:10,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:10,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:37:10,470 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:37:10,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:11,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:12,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:13,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:14,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:15,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:16,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:16,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:17,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:18,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:19,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:19,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:37:19,981 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:37:19,981 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:37:19,982 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:37:20,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:20,683 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:37:21,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:22,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:22,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:22,685 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:37:23,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:24,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:25,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:25,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:37:25,470 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:37:26,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:27,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:27,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:28,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:29,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:30,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:31,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:32,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:33,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:33,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:34,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:35,275 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:37:35,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:36,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:37,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:38,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:38,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:39,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:40,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:40,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:37:40,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:37:41,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:42,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:43,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:43,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:44,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:45,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:46,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:47,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:48,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:48,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:37:48,794 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:37:48,794 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:37:48,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:48,796 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:37:49,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:49,706 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:37:50,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:50,708 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:37:51,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:52,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:53,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:54,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:54,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:37:55,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:37:55,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:37:55,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:56,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:57,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:58,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:59,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:37:59,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:00,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:01,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:02,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:03,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:04,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:05,280 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:38:05,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:05,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:06,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:07,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:08,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:09,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:10,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:38:10,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:38:10,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:10,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:11,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:12,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:13,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:14,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:15,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:16,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:16,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:17,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:18,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:19,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:19,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:38:19,695 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:38:19,695 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:38:19,697 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:38:19,734 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:38:20,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:20,735 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:38:21,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:22,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:22,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:23,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:24,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:25,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:38:25,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:38:25,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:26,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:27,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:27,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:28,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:29,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:30,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:31,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:32,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:33,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:33,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:34,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:35,284 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:38:35,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:36,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:37,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:38,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:39,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:39,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:40,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:38:40,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:38:40,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:41,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:42,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:43,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:44,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:44,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:45,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:46,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:47,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:48,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:48,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:38:48,934 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:38:48,935 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:38:48,936 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:38:49,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:49,766 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:38:49,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:50,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:50,767 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:38:51,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:52,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:53,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:54,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:55,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:38:55,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:38:55,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:55,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:38:56,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:57,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:58,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:38:59,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:00,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:01,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:01,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:02,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:03,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:04,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:05,288 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:39:05,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:06,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:06,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:07,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:08,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:09,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:10,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:39:10,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:39:10,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:11,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:11,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:12,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:13,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:14,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:15,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:16,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:17,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:17,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:18,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:19,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:19,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:39:19,666 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:39:19,667 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:39:19,668 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:39:19,793 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:39:20,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:20,794 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:39:21,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:22,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:23,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:23,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:24,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:25,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:39:25,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:39:25,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:26,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:27,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:28,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:28,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:29,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:30,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:31,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:32,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:33,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:34,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:34,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:35,292 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:39:35,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:36,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:37,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:38,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:39,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:40,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:39:40,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:39:40,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:40,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:41,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:42,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:43,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:44,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:45,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:46,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:46,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:47,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:48,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:49,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:50,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:51,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:39:51,256 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:39:51,256 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:39:51,258 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:39:51,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:51,821 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:39:52,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:52,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:52,822 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:39:53,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:54,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:55,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:39:55,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:39:55,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:56,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:57,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:39:57,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:58,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:39:59,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:00,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:01,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:02,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:03,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:03,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:04,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:05,296 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:40:05,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:06,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:07,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:08,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:08,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:09,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:10,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:40:10,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:40:10,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:11,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:12,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:13,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:13,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:14,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:15,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:16,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:17,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:18,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:19,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:19,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:20,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:21,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:21,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:40:21,828 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:40:21,828 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:40:21,829 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:40:21,848 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:40:22,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:22,849 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:40:23,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:24,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:24,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:25,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:40:25,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:40:25,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:26,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:27,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:28,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:29,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:30,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:30,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:31,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:32,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:33,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:34,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:35,300 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:40:35,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:36,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:36,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:37,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:38,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:39,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:40,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:40:40,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:40:40,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:41,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:41,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:42,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:43,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:44,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:45,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:46,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:47,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:47,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:48,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:49,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:50,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:51,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:52,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:40:52,064 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:40:52,064 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:40:52,065 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:40:52,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:52,879 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:40:53,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:53,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:54,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:54,880 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:40:55,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:40:55,472 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:40:55,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:56,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:57,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:58,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:40:58,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:40:59,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:00,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:01,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:02,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:03,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:04,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:04,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:05,304 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:41:05,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:06,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:07,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:08,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:09,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:09,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:10,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:41:10,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:41:10,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:11,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:12,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:13,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:14,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:14,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:15,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:16,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:17,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:18,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:19,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:20,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:20,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:21,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:22,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:41:22,607 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:41:22,607 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:41:22,609 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:41:22,909 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:41:23,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:24,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:24,911 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:41:25,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:25,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:41:25,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:41:25,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:26,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:27,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:28,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:29,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:30,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:31,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:31,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:32,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:33,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:34,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:35,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:35,308 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:41:36,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:37,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:37,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:38,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:39,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:40,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:40,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:41:40,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:41:41,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:42,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:42,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:43,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:44,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:45,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:46,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:47,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:48,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:48,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:49,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:50,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:51,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:52,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:53,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:53,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:54,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:54,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:41:54,887 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:41:54,888 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:41:54,889 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:41:54,931 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:41:55,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:55,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:41:55,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:41:56,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:56,933 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:41:57,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:58,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:41:58,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:41:59,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:00,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:01,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:02,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:03,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:04,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:04,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:05,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:05,312 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:42:06,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:07,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:08,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:09,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:09,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:10,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:10,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:42:10,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:42:11,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:12,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:13,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:14,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:14,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:15,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:16,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:17,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:18,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:19,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:20,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:20,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:21,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:22,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:23,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:24,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:25,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:25,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:42:25,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:42:25,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:26,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:27,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:27,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:42:27,930 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:42:27,930 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:42:27,932 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:42:27,962 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:42:28,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:28,963 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:42:29,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:30,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:31,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:31,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:32,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:33,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:34,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:35,313 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:42:35,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:36,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:37,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:37,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:38,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:39,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:40,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:40,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:42:40,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:42:41,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:42,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:42,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:43,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:44,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:45,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:46,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:47,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:48,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:48,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:49,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:50,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:51,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:52,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:53,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:54,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:54,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:55,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:42:55,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:42:55,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:56,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:57,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:58,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:42:59,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:42:59,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:00,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:01,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:02,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:43:02,207 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:43:02,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:43:02,210 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:43:02,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:02,990 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:43:03,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:04,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:04,991 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:43:05,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:05,317 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:43:05,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:06,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:07,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:08,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:09,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:10,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:43:10,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:43:10,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:10,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:11,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:12,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:13,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:14,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:15,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:16,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:16,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:17,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:18,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:19,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:20,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:21,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:22,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:22,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:23,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:24,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:25,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:43:25,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:43:25,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:26,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:27,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:27,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:28,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:29,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:30,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:31,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:32,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:33,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:33,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:43:33,541 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:43:33,542 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:43:33,547 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:43:33,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:34,018 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:43:34,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:35,019 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:43:35,321 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:43:35,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:36,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:37,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:38,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:38,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:39,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:40,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:43:40,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:43:40,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:41,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:42,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:43,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:43,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:45,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:46,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:47,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:48,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:49,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:49,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:50,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:51,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:52,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:53,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:54,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:55,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:55,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:43:55,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:43:55,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:43:56,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:57,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:58,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:43:59,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:00,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:01,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:01,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:01,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:44:01,806 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:44:01,806 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:44:01,807 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:44:02,041 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:44:02,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:03,042 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:44:03,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:04,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:05,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:05,325 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:44:06,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:07,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:07,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:08,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:09,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:10,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:10,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:44:10,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:44:11,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:12,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:12,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:13,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:14,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:15,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:16,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:17,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:18,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:18,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:19,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:20,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:21,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:22,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:23,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:23,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:24,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:25,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:25,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:44:25,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:44:26,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:27,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:28,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:28,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:29,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:30,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:31,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:32,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:33,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:34,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:44:34,219 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:44:34,219 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:44:34,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:34,220 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:44:34,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:35,075 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:44:35,330 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:44:35,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:36,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:37,077 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:44:37,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:38,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:39,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:39,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:40,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:44:40,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:44:40,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:41,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:42,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:43,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:44,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:44,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:45,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:46,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:47,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:48,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:49,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:50,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:50,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:51,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:52,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:53,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:54,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:55,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:44:55,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:44:55,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:44:55,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:56,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:57,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:58,643 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:44:59,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:00,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:01,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:01,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:02,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:03,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:04,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:05,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:45:05,187 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:45:05,188 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:45:05,189 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:45:05,333 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:45:05,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:06,100 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:45:06,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:06,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:07,101 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:45:07,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:08,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:09,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:10,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:45:10,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:45:10,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:11,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:12,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:12,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:13,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:14,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:15,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:16,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:17,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:17,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:18,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:19,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:20,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:21,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:22,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:23,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:23,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:24,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:25,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:45:25,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:45:25,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:26,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:27,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:28,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:28,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:29,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:30,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:31,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:32,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:33,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:34,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:45:34,350 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:45:34,351 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:45:34,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:34,352 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:45:34,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:35,125 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:45:35,338 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:45:35,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:36,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:37,126 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:45:37,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:38,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:39,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:39,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:40,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:45:40,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:45:40,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:41,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:42,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:43,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:44,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:45,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:45,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:46,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:47,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:48,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:49,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:50,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:50,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:51,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:52,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:53,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:54,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:55,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:45:55,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:45:55,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:45:55,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:56,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:57,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:58,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:45:59,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:00,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:01,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:01,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:02,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:03,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:04,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:05,341 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:46:05,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:06,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:07,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:46:07,056 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:46:07,057 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:46:07,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:07,058 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:46:07,162 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:46:07,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:08,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:09,163 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:46:09,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:10,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:46:10,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:46:10,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:11,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:12,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:12,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:13,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:14,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:15,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:16,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:17,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:17,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:18,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:19,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:20,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:21,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:22,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:23,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:23,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:24,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:25,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:46:25,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:46:25,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:26,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:27,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:28,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:28,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:29,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:30,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:31,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:32,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:33,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:34,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:46:34,008 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:46:34,009 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:46:34,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:34,010 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:46:34,187 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:46:34,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:35,188 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:46:35,345 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:46:35,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:36,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:37,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:38,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:39,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:39,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:40,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:46:40,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:46:40,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:41,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:42,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:43,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:44,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:44,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:45,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:46,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:47,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:48,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:49,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:50,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:50,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:51,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:52,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:53,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:54,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:55,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:46:55,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:46:55,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:46:55,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:56,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:57,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:58,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:46:59,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:00,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:00,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:01,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:02,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:03,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:04,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:05,352 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:47:05,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:05,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:06,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:47:06,147 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:47:06,147 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:47:06,148 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:47:06,215 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:47:06,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:07,216 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:47:07,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:08,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:09,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:10,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:47:10,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:47:10,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:11,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:11,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:12,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:13,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:14,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:15,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:16,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:16,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:17,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:18,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:19,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:20,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:21,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:21,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:22,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:23,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:24,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:25,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:47:25,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:47:25,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:26,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:27,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:27,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:28,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:29,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:30,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:31,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:32,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:32,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:33,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:47:33,538 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:47:33,538 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:47:33,539 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:47:34,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:34,239 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:47:35,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:35,240 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:47:35,356 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:47:36,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:37,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:38,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:38,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:39,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:40,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:40,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:47:40,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:47:41,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:42,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:43,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:43,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:44,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:45,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:46,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:47,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:48,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:49,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:49,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:50,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:51,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:52,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:53,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:54,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:54,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:47:55,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:55,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:47:55,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:47:56,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:57,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:58,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:59,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:47:59,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:00,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:01,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:02,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:03,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:03,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:48:03,460 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:48:03,461 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:48:03,462 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:48:04,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:04,267 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:48:05,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:05,268 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:48:05,360 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:48:05,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:06,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:07,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:08,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:09,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:10,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:10,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:48:10,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:48:10,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:11,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:12,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:13,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:14,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:15,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:15,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:16,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:17,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:18,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:19,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:20,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:20,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:21,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:22,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:23,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:24,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:25,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:25,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:48:25,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:48:25,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:26,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:27,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:28,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:29,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:30,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:31,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:31,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:48:31,466 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:48:31,466 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:48:31,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:31,468 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:48:32,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:32,293 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:48:33,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:33,294 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:48:34,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:35,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:35,364 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:48:36,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:36,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:37,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:38,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:39,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:40,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:40,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:48:40,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:48:41,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:41,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:42,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:43,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:44,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:45,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:46,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:47,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:47,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:48,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:49,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:50,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:51,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:52,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:53,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:53,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:54,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:55,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:55,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:48:55,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:48:56,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:57,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:58,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:48:58,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:48:59,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:00,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:01,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:02,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:03,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:49:03,006 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:49:03,006 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:49:03,008 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:49:03,323 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:49:03,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:04,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:04,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:05,325 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:49:05,368 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:49:05,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:06,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:07,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:08,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:09,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:09,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:10,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:10,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:49:10,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:49:11,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:12,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:13,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:14,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:15,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:15,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:16,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:17,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:18,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:19,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:20,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:21,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:21,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:22,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:23,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:24,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:25,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:49:25,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:49:25,540 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:26,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:26,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:27,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:28,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:29,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:30,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:31,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:31,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:31,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:49:31,948 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:49:31,949 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:49:31,950 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:49:32,346 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:49:32,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:33,347 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:49:33,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:34,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:35,372 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:49:35,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:36,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:37,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:37,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:38,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:39,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:40,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:49:40,478 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:49:40,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:41,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:42,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:43,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:43,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:44,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:45,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:46,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:47,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:48,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:48,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:49,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:50,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:51,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:52,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:53,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:54,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:54,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:55,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:49:55,494 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:49:55,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:56,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:57,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:58,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:49:59,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:49:59,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:00,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:01,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:50:01,156 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:50:01,156 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:50:01,158 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:50:01,376 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:50:01,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:02,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:03,378 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:50:03,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:04,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:05,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:05,375 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:50:05,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:06,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:07,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:08,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:09,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:10,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:50:10,481 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:50:10,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:10,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:11,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:12,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:13,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:14,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:15,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:15,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:16,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:17,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:18,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:19,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:20,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:21,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:21,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:22,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:23,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:24,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:25,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:50:25,481 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:50:25,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:26,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:27,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:27,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:28,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:29,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:30,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:31,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:32,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:32,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:32,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:50:32,795 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:50:32,795 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:50:32,797 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:50:33,409 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:50:33,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:34,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:35,380 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:50:35,411 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:50:35,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:36,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:37,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:38,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:38,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:39,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:40,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:50:40,481 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:50:40,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:41,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:42,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:43,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:43,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:44,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:45,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:46,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:47,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:48,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:48,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:49,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:50,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:51,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:52,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:53,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:54,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:54,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:55,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:50:55,481 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:50:55,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:56,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:57,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:58,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:50:59,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:50:59,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:00,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:01,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:02,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:03,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:03,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:51:03,909 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:51:03,910 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:51:03,911 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:51:04,433 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:51:04,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:04,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:05,384 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:51:05,434 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:51:05,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:06,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:07,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:08,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:09,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:10,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:51:10,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:51:10,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:10,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:11,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:12,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:13,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:14,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:15,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:15,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:16,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:17,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:18,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:19,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:20,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:20,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:21,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:22,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:23,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:24,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:25,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:51:25,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:51:25,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:26,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:26,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:27,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:28,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:29,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:30,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:31,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:31,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:32,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:33,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:33,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:51:33,849 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:51:33,849 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:51:33,851 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:51:34,467 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:51:34,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:35,388 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:51:35,468 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:51:35,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:36,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:37,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:37,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:38,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:39,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:40,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:51:40,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:51:40,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:41,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:42,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:43,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:43,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:44,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:45,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:46,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:47,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:48,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:48,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:49,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:50,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:51,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:52,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:53,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:53,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:54,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:55,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:51:55,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:51:55,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:56,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:57,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:58,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:51:59,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:51:59,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:00,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:01,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:02,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:03,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:04,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:04,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:05,393 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:52:05,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:06,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:07,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:08,865 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:09,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:52:09,365 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:52:09,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:52:09,366 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:52:09,494 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:52:09,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:10,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:10,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:52:10,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:52:10,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:11,495 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:52:11,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:12,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:13,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:14,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:15,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:15,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:16,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:17,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:18,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:19,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:20,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:20,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:22,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:23,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:24,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:25,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:25,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:52:25,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:52:25,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:26,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:27,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:28,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:29,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:30,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:31,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:31,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:32,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:33,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:34,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:35,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:35,401 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:52:36,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:36,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:37,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:38,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:39,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:40,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:40,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:52:40,366 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:52:40,366 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:52:40,368 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:52:40,517 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:52:41,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:52:41,431 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:52:41,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:41,519 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:52:42,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:42,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:43,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:44,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:45,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:46,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:47,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:48,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:48,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:49,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:50,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:51,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:52,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:53,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:54,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:54,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:55,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:55,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:52:55,515 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:52:56,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:57,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:58,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:52:59,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:52:59,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:00,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:01,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:02,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:03,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:04,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:04,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:05,402 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:53:05,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:06,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:07,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:08,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:09,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:09,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:10,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:53:10,417 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:53:10,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:53:10,418 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:53:10,541 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:53:11,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:53:11,499 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:53:11,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:11,542 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:53:11,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:12,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:13,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:14,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:14,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:15,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:16,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:17,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:18,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:19,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:20,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:20,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:21,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:22,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:23,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:24,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:25,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:53:25,566 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:53:25,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:25,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:26,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:27,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:28,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:29,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:30,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:31,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:31,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:32,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:33,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:34,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:35,403 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:53:35,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:36,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:36,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:37,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:38,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:39,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:40,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:53:40,569 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:53:40,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:41,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:41,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:53:41,885 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:53:41,885 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:53:41,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:41,889 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:53:42,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:42,573 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:53:43,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:43,574 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:53:44,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:45,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:46,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:47,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:47,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:48,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:49,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:50,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:51,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:52,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:52,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:53,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:54,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:55,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:53:55,567 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:53:55,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:56,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:57,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:53:57,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:58,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:53:59,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:00,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:01,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:02,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:03,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:03,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:04,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:05,406 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:54:05,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:06,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:07,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:08,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:08,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:09,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:10,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:54:10,569 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:54:10,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:11,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:12,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:13,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:54:13,444 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:54:13,444 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:54:13,448 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:54:13,610 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:54:13,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:14,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:14,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:15,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:15,612 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:54:16,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:17,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:18,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:19,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:19,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:20,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:21,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:22,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:23,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:24,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:25,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:25,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:54:25,569 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:54:25,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:26,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:27,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:28,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:29,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:30,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:30,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:31,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:32,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:33,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:34,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:35,410 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:54:35,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:36,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:36,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:37,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:38,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:39,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:40,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:54:40,572 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:54:40,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:41,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:41,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:42,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:43,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:44,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:54:44,103 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:54:44,104 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:54:44,109 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:54:44,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:44,641 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:54:45,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:45,642 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:54:46,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:47,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:47,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:48,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:49,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:50,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:51,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:52,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:53,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:53,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:54,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:55,571 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:54:55,572 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:54:55,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:56,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:57,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:58,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:54:59,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:54:59,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:00,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:01,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:02,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:03,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:04,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:04,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:05,416 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:55:05,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:06,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:07,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:08,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:09,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:09,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:10,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:55:10,576 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:55:10,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:11,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:12,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:13,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:13,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:55:13,927 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:55:13,927 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:55:13,928 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:55:14,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:14,666 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:55:14,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:15,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:15,667 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:55:16,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:17,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:18,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:19,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:20,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:20,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:21,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:22,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:23,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:24,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:25,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:25,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:55:25,572 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:55:25,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:26,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:27,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:28,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:29,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:30,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:31,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:31,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:32,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:33,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:34,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:35,418 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:55:35,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:36,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:37,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:37,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:38,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:39,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:40,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:55:40,575 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:55:40,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:41,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:42,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:42,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:43,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:44,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:55:44,354 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:55:44,354 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:55:44,356 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:55:44,704 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:55:44,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:45,705 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:55:45,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:46,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:47,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:48,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:48,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:49,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:50,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:51,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:52,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:53,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:53,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:54,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:55,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:55:55,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:55:55,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:56,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:57,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:58,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:55:59,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:55:59,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:00,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:01,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:02,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:03,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:04,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:04,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:05,422 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:56:05,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:06,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:07,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:08,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:09,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:10,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:10,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:56:10,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:56:10,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:11,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:12,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:13,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:14,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:15,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:15,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:16,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:17,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:18,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:19,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:56:19,416 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:56:19,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:56:19,419 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:56:19,742 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:56:19,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:20,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:21,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:21,744 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:56:21,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:22,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:23,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:24,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:25,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:56:25,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:56:25,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:26,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:26,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:27,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:28,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:29,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:30,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:31,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:32,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:32,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:33,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:34,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:35,426 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:56:35,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:36,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:37,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:37,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:38,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:39,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:40,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:56:40,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:56:40,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:41,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:42,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:43,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:43,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:44,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:45,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:46,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:47,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:48,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:48,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:49,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:50,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:51,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:56:51,545 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:56:51,546 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:56:51,551 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:56:51,771 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:56:51,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:52,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:53,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:53,773 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:56:53,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:54,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:55,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:56:55,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:56:55,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:56,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:57,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:58,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:56:59,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:56:59,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:00,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:01,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:02,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:03,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:04,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:05,431 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:57:05,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:05,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:06,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:07,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:08,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:09,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:10,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:10,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:57:10,574 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:57:10,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:11,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:12,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:13,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:14,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:15,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:15,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:16,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:17,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:18,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:19,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:20,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:20,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:21,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:22,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:23,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:57:23,740 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:57:23,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:57:23,746 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:57:23,801 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:57:24,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:25,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:25,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:57:25,575 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:57:25,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:25,803 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:57:26,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:27,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:28,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:29,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:30,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:31,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:31,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:32,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:33,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:34,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:35,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:35,435 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:57:36,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:37,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:37,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:38,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:39,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:40,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:40,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:57:40,575 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:57:41,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:42,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:42,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:43,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:44,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:45,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:46,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:47,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:48,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:48,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:49,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:50,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:51,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:52,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:53,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:53,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:57:54,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:55,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:55,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:57:55,578 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:57:55,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:57:55,717 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:57:55,718 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:57:55,720 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:57:55,829 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:57:56,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:57,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:57,831 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:57:58,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:59,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:57:59,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:00,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:01,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:02,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:03,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:04,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:05,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:05,439 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:58:05,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:06,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:07,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:08,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:09,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:10,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:10,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:10,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:58:10,577 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:58:11,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:12,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:13,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:14,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:15,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:16,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:16,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:17,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:18,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:19,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:20,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:21,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:21,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:22,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:23,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:24,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:25,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:25,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:58:25,578 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:58:25,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:58:25,789 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:58:25,789 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:58:25,795 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:58:25,862 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:58:26,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:27,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:27,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:27,864 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:58:28,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:29,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:30,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:31,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:32,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:33,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:33,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:34,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:35,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:35,443 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:58:36,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:37,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:38,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:39,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:39,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:40,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:40,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:58:40,829 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:58:41,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:42,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:43,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:44,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:44,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:45,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:46,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:47,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:48,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:49,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:50,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:50,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:51,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:52,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:53,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:54,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:55,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:55,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:58:55,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:58:55,599 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:58:56,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:57,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:58,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:59,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:58:59,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:58:59,855 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:58:59,855 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:58:59,860 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:58:59,893 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:59:00,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:00,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:01,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:01,895 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:59:02,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:03,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:04,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:05,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:05,447 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:59:06,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:06,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:07,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:08,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:09,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:10,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:10,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:59:10,597 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:59:11,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:11,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:12,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:13,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:14,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:15,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:16,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:17,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:17,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:18,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:19,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:20,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:21,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:22,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:22,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:23,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:24,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:25,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:25,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:59:25,599 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:59:26,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:27,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:59:27,187 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:59:27,187 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:59:27,188 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:59:27,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:27,921 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:59:27,922 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:59:28,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:28,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:29,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:30,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:31,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:32,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:33,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:33,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:34,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:35,451 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 18:59:35,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:36,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:37,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:38,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:39,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:39,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:40,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:59:40,599 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:59:40,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:41,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:42,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:43,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:44,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:45,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:45,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:46,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:47,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:48,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:49,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:50,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:51,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:51,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:52,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:53,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:54,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:55,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 18:59:55,608 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 18:59:55,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:56,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 18:59:56,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:56,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 18:59:56,851 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 18:59:56,851 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 18:59:56,853 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 18:59:56,965 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 18:59:57,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:57,967 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 18:59:58,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 18:59:59,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:00,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:01,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:02,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:02,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:03,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:04,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:05,456 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:00:05,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:06,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:07,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:08,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:08,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:09,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:10,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:00:10,602 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:00:10,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:11,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:12,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:13,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:13,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:14,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:15,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:16,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:17,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:18,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:18,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:19,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:20,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:21,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:22,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:23,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:23,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:24,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:25,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:00:25,612 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:00:25,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:26,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:26,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:00:26,867 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:00:26,868 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:00:26,869 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:00:26,998 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:00:27,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:28,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:28,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:29,000 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:00:29,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:30,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:31,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:32,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:33,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:34,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:34,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:35,460 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:00:35,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:36,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:37,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:38,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:39,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:39,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:40,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:00:40,606 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:00:40,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:41,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:42,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:43,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:44,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:45,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:45,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:46,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:47,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:48,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:49,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:50,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:50,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:51,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:52,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:53,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:54,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:55,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:00:55,607 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:00:55,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:55,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:00:55,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:00:55,905 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:00:55,905 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:00:55,907 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:00:56,027 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:00:56,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:57,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:58,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:00:59,030 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:00:59,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:00,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:01,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:01,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:02,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:03,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:04,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:05,464 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:01:05,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:06,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:06,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:07,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:08,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:09,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:10,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:01:10,606 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:01:10,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:11,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:11,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:12,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:13,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:14,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:15,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:16,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:16,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:17,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:18,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:19,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:20,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:21,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:21,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:22,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:23,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:24,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:25,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:01:25,607 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:01:25,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:25,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:01:25,833 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:01:25,833 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:01:25,835 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:01:26,056 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:01:26,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:27,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:27,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:28,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:29,058 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:01:29,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:30,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:31,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:32,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:33,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:33,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:34,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:35,468 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:01:35,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:36,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:37,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:38,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:39,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:39,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:40,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:01:40,607 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:01:40,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:41,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:42,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:43,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:44,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:44,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:45,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:46,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:47,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:48,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:49,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:50,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:50,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:51,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:52,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:53,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:54,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:55,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:01:55,613 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:01:55,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:55,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:01:56,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:01:56,516 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:01:56,517 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:01:56,518 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:01:56,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:57,089 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:01:57,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:58,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:01:59,091 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:01:59,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:00,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:00,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:01,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:02,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:03,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:04,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:05,472 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:02:05,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:06,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:06,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:07,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:08,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:09,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:10,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:02:10,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:02:10,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:11,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:11,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:12,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:13,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:14,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:15,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:16,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:17,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:17,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:18,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:19,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:20,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:21,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:22,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:22,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:23,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:24,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:24,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:02:24,798 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:02:24,799 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:02:24,801 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:02:25,116 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:02:25,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:02:25,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:02:25,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:26,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:27,118 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:02:27,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:27,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:28,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:29,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:30,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:31,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:32,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:33,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:33,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:34,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:35,476 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:02:35,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:36,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:37,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:38,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:38,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:39,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:40,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:02:40,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:02:40,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:41,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:42,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:43,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:44,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:44,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:45,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:46,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:47,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:48,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:49,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:49,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:50,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:51,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:52,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:53,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:54,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:02:54,403 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:02:54,404 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:02:54,405 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:02:54,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:55,145 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:02:55,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:02:55,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:02:55,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:02:55,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:56,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:57,147 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:02:57,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:58,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:02:59,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:00,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:00,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:01,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:02,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:03,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:04,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:05,481 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:03:06,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:06,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:07,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:08,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:09,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:10,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:10,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:03:10,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:03:11,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:11,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:12,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:13,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:14,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:15,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:16,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:16,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:17,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:18,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:19,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:20,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:21,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:21,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:21,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:03:21,950 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:03:21,951 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:03:21,952 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:03:22,173 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:03:22,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:23,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:24,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:25,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:25,175 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:03:25,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:03:25,610 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:03:26,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:27,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:27,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:28,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:29,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:30,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:31,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:32,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:33,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:33,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:34,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:35,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:35,484 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:03:36,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:37,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:38,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:38,828 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:39,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:40,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:40,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:03:40,610 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:03:41,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:42,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:43,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:44,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:44,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:45,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:46,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:47,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:48,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:49,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:49,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:50,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:51,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:52,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:53,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:54,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:54,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:03:54,230 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:03:54,230 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:03:54,232 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:03:55,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:55,204 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:03:55,205 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:03:55,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:03:55,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:03:55,610 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:03:56,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:57,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:58,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:03:59,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:00,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:00,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:01,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:02,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:03,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:04,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:05,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:05,488 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:04:06,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:06,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:07,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:08,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:09,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:10,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:10,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:04:10,610 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:04:11,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:11,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:12,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:13,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:14,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:15,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:16,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:16,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:17,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:18,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:19,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:20,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:21,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:21,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:22,128 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:23,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:24,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:24,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:04:24,808 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:04:24,809 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:04:24,811 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:04:25,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:25,235 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:04:25,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:04:25,610 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:04:26,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:26,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:27,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:27,237 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:04:28,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:29,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:30,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:31,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:31,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:32,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:33,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:34,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:35,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:35,491 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:04:36,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:37,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:37,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:38,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:39,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:40,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:40,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:04:40,610 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:04:41,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:42,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:42,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:43,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:44,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:45,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:46,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:47,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:48,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:48,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:49,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:50,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:51,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:52,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:53,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:53,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:04:54,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:55,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:55,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:04:55,463 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:04:55,463 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:04:55,465 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:04:55,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:04:55,880 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:04:56,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:56,269 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:04:57,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:57,270 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:04:58,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:59,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:04:59,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:00,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:01,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:02,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:03,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:04,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:05,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:05,496 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:05:05,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:06,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:07,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:08,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:09,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:10,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:10,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:05:10,616 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:05:10,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:11,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:12,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:13,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:14,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:15,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:15,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:16,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:17,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:18,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:19,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:20,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:20,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:21,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:22,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:23,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:24,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:25,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:25,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:05:25,467 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:05:25,467 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:05:25,468 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:05:25,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:05:25,891 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:05:26,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:26,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:26,296 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:05:27,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:27,297 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:05:28,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:29,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:30,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:31,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:31,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:32,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:33,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:34,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:35,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:35,500 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:05:36,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:37,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:37,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:38,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:39,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:40,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:40,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:05:40,617 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:05:41,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:42,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:42,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:43,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:44,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:45,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:46,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:47,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:48,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:48,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:49,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:50,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:51,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:52,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:53,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:53,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:05:54,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:55,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:55,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:05:55,618 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:05:55,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:05:55,801 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:05:55,801 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:05:55,802 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:05:56,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:56,324 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:05:57,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:57,325 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:05:58,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:59,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:05:59,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:00,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:01,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:02,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:03,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:04,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:05,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:05,505 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:06:05,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:06,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:07,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:08,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:09,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:10,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:10,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:10,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:06:10,917 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:06:11,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:12,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:13,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:14,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:15,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:15,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:16,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:17,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:18,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:19,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:20,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:20,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:21,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:22,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:23,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:24,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:25,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:25,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:25,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:06:25,920 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:06:26,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:27,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:28,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:29,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:06:29,151 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:06:29,151 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:06:29,153 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:06:29,363 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:06:29,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:30,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:31,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:31,365 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:06:31,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:32,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:33,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:34,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:35,508 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:06:35,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:36,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:36,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:37,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:38,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:39,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:40,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:40,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:06:40,920 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:06:41,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:41,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:42,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:43,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:44,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:45,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:46,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:46,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:47,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:48,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:49,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:50,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:51,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:52,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:52,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:53,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:54,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:55,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:55,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:06:55,927 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:06:56,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:57,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:57,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:06:58,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:06:59,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:00,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:00,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:07:00,824 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:07:00,825 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:07:00,826 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:07:01,390 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:07:01,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:02,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:03,392 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:07:03,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:03,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:04,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:05,512 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:07:05,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:06,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:07,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:08,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:09,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:09,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:10,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:10,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:07:10,922 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:07:11,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:12,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:13,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:14,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:14,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:15,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:16,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:17,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:18,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:19,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:20,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:20,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:21,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:22,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:23,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:24,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:25,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:25,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:25,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:07:25,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:07:26,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:27,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:28,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:29,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:30,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:30,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:31,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:31,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:07:31,595 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:07:31,596 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:07:31,597 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:07:32,418 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:07:32,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:33,419 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:07:33,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:34,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:35,515 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:07:35,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:36,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:36,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:37,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:38,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:39,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:40,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:40,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:07:40,926 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:07:41,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:42,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:42,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:43,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:44,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:45,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:46,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:47,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:47,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:48,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:49,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:50,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:51,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:52,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:53,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:53,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:54,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:55,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:55,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:07:55,935 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:07:56,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:57,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:58,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:07:58,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:07:59,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:00,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:01,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:08:01,359 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:08:01,360 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:08:01,361 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:08:01,443 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:08:01,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:02,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:03,445 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:08:03,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:04,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:04,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:05,519 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:08:05,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:06,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:07,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:08,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:09,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:10,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:10,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:10,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:08:10,931 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:08:11,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:12,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:13,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:14,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:15,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:15,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:16,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:17,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:18,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:19,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:20,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:20,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:21,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:22,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:23,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:24,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:25,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:25,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:25,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:08:25,931 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:08:26,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:27,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:28,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:29,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:30,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:31,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:31,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:32,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:33,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:34,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:35,521 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:08:35,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:35,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:08:35,939 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:08:35,939 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:08:35,941 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:08:36,475 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:08:36,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:36,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:37,476 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:08:37,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:38,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:39,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:40,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:40,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:08:40,931 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:08:41,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:42,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:42,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:43,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:44,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:45,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:46,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:47,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:47,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:48,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:49,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:50,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:51,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:52,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:52,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:53,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:54,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:55,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:55,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:08:55,931 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:08:56,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:57,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:58,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:08:58,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:08:59,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:00,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:01,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:02,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:03,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:03,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:04,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:05,522 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:09:05,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:06,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:09:06,364 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:09:06,364 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:09:06,366 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:09:06,504 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:09:06,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:07,505 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:09:07,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:08,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:09,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:09,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:10,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:10,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:09:10,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:09:11,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:12,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:13,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:14,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:14,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:15,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:16,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:17,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:18,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:19,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:20,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:20,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:21,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:22,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:23,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:24,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:25,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:25,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:09:25,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:09:26,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:26,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:27,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:28,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:29,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:30,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:31,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:32,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:32,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:33,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:34,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:35,525 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:09:35,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:09:35,609 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:09:35,609 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:09:35,611 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:09:36,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:36,530 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:09:37,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:37,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:37,531 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:09:38,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:39,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:40,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:40,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:09:40,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:09:41,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:42,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:42,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:43,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:44,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:45,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:46,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:47,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:48,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:48,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:49,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:50,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:51,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:52,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:53,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:53,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:09:54,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:55,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:55,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:09:55,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:09:56,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:57,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:58,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:59,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:09:59,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:00,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:01,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:02,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:03,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:04,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:04,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:05,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:05,531 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:10:06,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:07,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:07,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:10:07,284 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:10:07,285 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:10:07,287 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:10:07,557 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:10:08,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:09,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:09,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:09,559 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:10:10,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:10,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:10:10,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:10:11,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:12,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:13,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:14,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:15,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:15,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:16,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:17,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:18,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:19,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:20,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:20,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:21,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:22,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:23,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:24,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:25,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:25,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:25,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:10:25,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:10:26,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:27,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:28,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:29,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:30,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:30,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:31,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:32,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:33,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:34,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:34,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:10:34,953 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:10:34,953 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:10:34,955 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:10:35,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:35,531 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:10:35,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:35,584 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:10:36,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:37,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:37,585 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:10:38,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:39,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:40,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:40,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:10:40,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:10:41,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:41,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:42,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:43,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:44,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:45,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:46,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:47,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:47,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:48,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:49,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:50,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:51,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:52,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:53,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:53,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:54,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:55,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:55,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:10:55,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:10:56,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:57,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:58,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:10:59,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:10:59,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:00,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:01,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:02,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:03,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:04,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:04,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:11:04,135 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:11:04,135 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:11:04,137 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:11:04,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:04,614 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:11:05,535 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:11:05,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:05,615 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:11:06,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:07,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:08,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:09,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:09,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:10,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:10,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:11:10,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:11:11,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:12,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:13,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:14,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:14,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:15,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:16,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:17,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:18,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:19,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:19,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:20,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:21,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:22,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:23,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:24,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:25,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:25,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:25,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:11:25,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:11:26,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:27,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:28,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:29,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:30,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:30,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:31,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:32,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:11:32,430 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:11:32,431 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:11:32,436 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:11:32,645 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:11:32,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:33,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:33,646 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:11:34,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:35,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:35,539 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:11:35,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:36,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:37,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:38,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:39,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:40,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:40,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:11:40,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:11:41,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:41,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:42,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:43,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:44,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:45,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:46,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:46,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:47,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:48,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:49,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:50,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:51,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:52,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:52,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:53,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:54,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:55,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:55,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:11:55,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:11:56,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:57,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:11:57,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:58,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:11:59,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:00,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:12:00,556 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:12:00,556 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:12:00,557 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:12:00,670 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:12:00,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:01,671 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:12:01,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:02,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:03,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:03,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:04,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:05,543 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:12:05,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:06,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:07,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:08,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:09,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:09,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:10,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:12:10,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:12:10,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:11,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:12,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:13,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:14,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:15,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:15,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:16,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:17,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:18,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:19,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:20,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:20,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:21,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:22,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:23,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:24,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:25,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:12:25,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:12:25,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:26,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:26,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:27,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:28,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:29,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:30,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:31,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:32,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:32,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:33,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:12:33,876 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:12:33,876 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:12:33,878 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:12:34,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:34,701 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:12:35,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:35,547 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:12:35,702 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:12:36,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:37,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:38,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:38,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:39,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:40,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:40,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:12:40,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:12:41,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:42,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:43,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:43,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:44,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:45,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:46,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:47,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:48,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:48,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:49,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:50,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:51,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:52,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:53,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:53,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:54,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:55,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:55,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:12:55,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:12:56,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:57,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:58,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:12:58,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:12:59,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:00,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:01,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:02,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:02,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:13:02,365 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:13:02,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:13:02,367 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:13:02,738 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:13:03,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:03,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:03,739 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:13:04,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:05,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:05,551 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:13:06,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:07,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:08,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:09,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:09,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:10,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:10,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:13:10,933 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:13:11,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:12,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:13,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:14,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:15,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:15,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:16,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:17,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:18,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:19,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:20,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:20,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:21,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:22,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:23,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:24,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:25,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:25,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:13:25,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:13:26,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:26,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:27,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:28,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:29,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:30,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:31,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:32,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:32,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:33,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:34,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:34,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:13:34,335 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:13:34,336 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:13:34,337 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:13:34,767 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:13:35,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:35,555 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:13:35,767 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:13:36,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:37,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:38,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:38,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:39,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:40,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:40,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:13:40,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:13:41,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:42,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:43,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:44,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:44,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:45,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:46,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:47,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:48,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:49,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:49,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:50,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:51,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:52,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:53,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:54,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:55,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:55,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:13:55,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:13:55,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:13:56,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:57,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:58,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:13:59,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:00,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:00,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:01,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:02,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:03,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:03,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:14:03,764 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:14:03,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:14:03,767 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:14:03,793 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:14:04,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:05,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:05,561 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:14:05,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:05,795 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:14:06,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:07,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:08,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:09,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:10,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:10,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:14:10,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:14:11,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:11,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:12,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:13,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:14,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:15,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:16,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:16,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:17,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:18,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:19,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:20,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:21,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:21,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:22,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:23,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:24,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:25,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:25,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:14:25,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:14:26,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:27,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:27,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:28,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:29,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:30,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:31,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:32,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:32,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:33,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:14:33,085 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:14:33,086 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:14:33,087 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:14:33,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:33,821 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:14:33,821 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:14:34,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:35,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:35,564 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:14:36,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:37,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:37,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:38,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:39,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:40,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:40,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:14:40,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:14:41,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:42,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:43,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:43,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:44,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:45,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:46,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:47,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:48,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:48,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:49,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:50,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:51,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:52,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:53,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:54,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:54,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:55,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:55,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:14:55,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:14:56,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:57,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:58,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:14:59,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:14:59,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:00,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:01,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:02,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:03,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:03,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:15:03,291 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:15:03,291 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:15:03,293 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:15:03,849 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:15:04,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:04,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:05,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:05,567 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:15:05,850 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:15:06,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:07,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:08,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:09,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:10,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:10,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:10,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:15:10,934 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:15:11,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:12,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:13,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:14,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:15,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:16,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:16,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:17,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:18,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:19,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:20,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:21,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:21,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:22,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:23,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:24,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:25,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:25,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:15:25,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:15:26,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:27,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:27,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:28,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:29,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:30,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:31,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:15:31,066 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:15:31,066 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:15:31,067 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:15:31,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:31,879 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:15:31,879 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:15:32,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:32,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:33,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:34,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:35,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:35,571 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:15:36,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:37,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:38,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:38,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:39,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:40,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:40,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:15:40,936 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:15:41,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:42,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:43,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:44,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:44,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:45,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:46,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:47,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:48,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:49,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:49,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:50,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:51,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:52,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:53,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:54,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:55,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:15:55,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:55,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:15:55,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:15:56,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:57,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:58,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:15:59,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:00,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:00,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:00,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:16:00,773 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:16:00,773 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:16:00,775 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:16:00,907 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:16:01,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:01,908 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:16:02,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:03,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:04,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:05,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:05,575 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:16:05,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:06,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:07,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:08,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:09,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:10,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:10,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:16:10,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:16:11,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:11,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:12,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:13,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:14,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:15,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:16,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:16,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:17,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:18,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:19,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:20,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:21,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:21,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:22,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:23,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:24,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:25,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:25,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:16:25,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:16:26,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:27,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:27,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:28,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:29,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:30,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:31,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:32,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:32,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:16:32,135 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:16:32,135 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:16:32,137 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:16:32,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:32,953 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:16:33,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:33,954 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:16:34,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:35,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:35,580 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:16:36,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:37,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:38,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:38,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:39,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:40,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:40,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:16:40,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:16:41,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:42,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:43,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:44,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:44,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:45,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:46,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:47,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:48,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:49,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:49,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:50,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:51,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:52,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:53,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:54,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:55,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:16:55,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:55,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:16:55,947 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:16:56,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:57,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:58,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:16:59,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:00,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:00,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:01,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:17:01,049 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:17:01,049 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:17:01,050 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:17:01,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:01,979 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:17:01,980 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:17:02,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:03,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:04,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:05,584 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:17:05,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:05,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:06,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:07,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:08,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:09,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:10,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:10,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:17:10,942 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:17:11,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:11,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:12,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:13,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:14,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:15,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:16,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:17,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:17,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:18,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:19,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:20,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:21,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:22,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:23,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:23,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:24,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:25,675 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:25,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:17:25,959 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:17:26,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:27,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:28,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:28,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:29,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:30,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:31,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:17:31,658 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:17:31,658 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:17:31,659 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:17:32,005 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:17:32,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:33,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:34,007 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:17:34,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:34,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:35,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:35,588 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:17:36,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:37,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:38,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:39,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:39,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:40,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:40,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:17:40,948 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:17:41,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:42,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:43,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:44,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:44,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:45,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:46,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:47,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:48,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:49,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:49,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:50,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:51,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:52,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:53,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:54,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:54,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:17:55,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:55,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:17:55,954 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:17:56,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:57,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:58,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:59,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:17:59,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:00,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:00,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:18:00,592 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:18:00,592 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:18:00,594 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:18:01,034 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:18:01,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:02,034 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:18:02,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:03,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:04,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:05,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:05,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:05,592 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:18:06,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:07,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:08,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:09,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:10,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:10,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:10,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:18:10,953 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:18:11,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:12,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:13,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:14,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:15,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:16,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:16,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:17,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:18,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:19,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:20,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:21,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:21,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:22,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:23,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:24,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:25,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:25,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:18:25,954 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:18:26,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:27,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:27,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:28,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:29,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:30,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:31,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:32,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:32,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:32,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:18:32,740 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:18:32,740 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:18:32,742 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:18:33,063 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:18:33,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:34,064 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:18:34,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:35,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:35,596 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:18:36,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:37,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:37,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:38,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:39,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:40,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:40,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:18:40,953 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:18:41,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:42,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:43,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:43,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:44,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:45,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:46,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:47,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:48,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:48,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:49,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:50,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:51,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:52,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:53,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:54,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:54,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:55,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:55,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:18:55,959 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:18:56,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:57,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:58,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:18:59,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:18:59,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:00,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:00,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:19:00,313 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:19:00,313 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:19:00,315 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:19:01,094 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:19:01,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:02,095 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:19:02,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:03,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:04,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:05,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:05,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:05,600 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:19:06,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:07,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:08,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:09,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:10,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:10,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:19:10,959 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:19:11,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:11,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:12,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:13,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:14,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:15,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:16,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:17,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:17,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:18,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:19,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:20,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:21,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:22,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:23,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:23,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:24,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:25,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:25,961 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:19:25,964 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:19:26,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:27,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:28,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:28,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:29,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:29,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:19:29,283 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:19:29,283 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:19:29,285 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:19:30,121 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:19:30,121 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:19:30,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:31,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:32,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:33,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:34,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:34,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:35,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:35,605 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:19:36,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:37,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:38,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:39,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:39,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:40,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:40,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:19:40,962 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:19:41,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:42,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:43,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:44,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:44,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:45,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:46,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:47,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:48,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:49,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:50,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:50,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:51,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:52,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:53,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:54,259 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:55,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:19:55,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:55,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:19:55,968 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:19:56,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:57,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:58,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:59,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:19:59,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:19:59,901 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:19:59,902 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:19:59,903 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:20:00,144 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:20:00,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:00,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:01,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:02,146 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:20:02,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:03,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:04,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:05,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:05,609 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:20:06,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:06,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:07,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:08,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:09,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:10,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:10,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:20:10,965 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:20:11,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:12,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:12,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:13,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:14,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:15,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:16,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:17,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:17,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:18,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:19,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:20,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:21,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:22,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:23,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:23,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:24,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:25,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:25,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:20:25,965 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:20:26,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:27,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:28,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:28,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:29,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:29,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:20:29,511 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:20:29,512 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:20:29,513 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:20:30,175 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:20:30,176 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:20:30,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:31,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:32,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:33,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:34,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:34,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:35,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:35,613 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:20:36,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:37,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:38,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:39,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:40,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:40,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:40,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:20:40,965 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:20:41,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:42,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:43,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:44,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:45,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:45,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:46,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:47,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:48,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:49,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:50,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:51,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:51,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:52,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:53,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:54,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:55,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:55,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:20:55,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:20:56,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:57,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:20:57,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:58,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:20:59,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:00,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:01,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:02,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:02,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:03,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:03,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:21:03,561 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:21:03,562 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:21:03,563 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:21:04,208 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:21:04,208 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:21:04,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:05,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:05,617 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:21:06,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:07,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:08,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:08,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:09,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:10,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:10,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:21:10,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:21:11,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:12,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:13,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:14,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:14,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:15,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:16,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:17,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:18,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:19,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:19,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:20,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:21,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:22,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:23,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:24,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:25,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:25,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:25,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:21:25,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:21:26,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:27,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:28,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:29,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:30,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:30,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:31,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:32,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:33,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:34,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:34,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:21:34,394 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:21:34,395 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:21:34,396 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:21:35,236 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:21:35,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:35,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:35,621 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:21:36,237 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:21:36,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:37,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:38,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:39,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:40,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:40,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:21:40,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:21:41,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:41,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:42,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:43,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:44,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:45,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:46,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:46,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:47,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:48,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:49,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:50,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:51,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:51,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:52,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:53,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:54,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:55,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:55,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:21:55,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:21:56,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:57,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:21:57,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:58,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:21:59,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:00,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:01,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:02,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:02,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:03,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:04,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:05,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:22:05,099 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:22:05,100 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:22:05,101 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:22:05,270 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:22:05,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:05,626 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:22:06,271 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:22:06,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:07,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:07,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:08,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:09,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:10,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:10,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:22:10,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:22:11,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:12,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:13,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:13,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:14,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:15,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:16,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:17,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:18,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:18,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:19,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:20,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:21,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:22,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:23,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:24,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:24,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:25,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:25,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:22:25,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:22:26,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:27,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:28,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:29,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:29,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:30,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:31,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:32,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:22:32,671 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:22:32,672 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:22:32,673 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:22:33,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:33,297 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:22:33,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:34,298 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:22:34,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:34,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:35,630 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:22:35,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:36,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:37,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:38,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:39,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:40,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:40,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:40,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:22:40,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:22:41,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:42,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:43,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:44,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:46,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:46,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:47,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:48,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:49,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:50,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:51,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:51,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:52,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:53,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:54,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:55,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:55,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:22:55,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:22:56,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:57,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:57,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:22:58,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:22:59,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:00,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:01,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:02,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:02,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:03,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:03,522 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:23:03,524 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:23:03,524 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:23:03,525 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:23:04,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:04,331 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:23:05,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:05,331 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:23:05,634 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:23:06,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:07,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:08,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:08,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:09,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:10,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:10,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:23:10,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:23:11,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:12,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:13,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:14,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:14,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:15,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:16,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:17,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:18,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:19,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:19,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:20,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:21,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:22,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:23,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:24,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:25,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:25,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:25,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:23:25,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:23:26,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:27,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:28,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:29,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:30,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:30,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:31,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:32,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:33,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:34,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:34,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:23:34,528 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:23:34,529 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:23:34,530 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:23:35,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:35,359 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:23:35,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:35,638 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:23:36,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:37,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:37,360 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:23:38,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:39,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:40,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:40,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:23:40,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:23:41,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:41,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:42,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:43,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:44,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:45,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:46,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:46,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:47,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:48,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:49,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:50,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:51,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:51,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:52,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:53,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:54,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:55,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:55,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:23:55,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:23:56,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:57,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:23:57,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:58,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:23:59,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:00,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:01,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:02,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:02,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:03,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:04,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:04,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:24:04,720 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:24:04,720 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:24:04,721 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:24:05,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:05,388 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:24:05,642 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:24:06,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:07,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:07,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:07,390 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:24:08,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:09,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:10,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:10,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:24:10,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:24:11,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:12,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:12,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:13,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:14,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:15,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:16,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:17,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:17,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:18,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:19,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:20,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:21,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:22,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:23,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:23,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:24,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:25,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:25,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:24:25,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:24:26,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:27,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:28,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:28,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:29,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:30,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:31,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:32,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:33,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:33,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:34,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:34,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:24:34,652 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:24:34,652 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:24:34,654 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:24:35,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:35,424 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:24:35,647 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:24:36,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:37,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:37,425 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:24:38,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:39,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:39,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:40,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:40,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:24:40,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:24:41,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:42,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:43,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:44,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:45,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:45,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:46,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:47,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:48,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:49,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:50,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:51,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:51,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:52,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:53,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:54,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:55,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:55,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:24:55,967 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:24:56,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:57,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:24:57,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:58,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:24:59,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:00,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:01,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:02,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:02,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:03,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:04,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:05,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:05,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:25:05,611 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:25:05,612 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:25:05,614 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:25:05,649 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:25:06,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:06,449 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:25:07,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:07,449 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:25:08,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:08,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:09,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:10,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:10,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:25:10,968 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:25:11,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:12,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:13,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:14,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:14,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:15,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:16,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:17,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:18,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:19,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:19,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:20,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:21,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:22,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:23,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:24,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:24,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:25,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:25,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:25:25,968 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:25:26,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:27,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:28,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:29,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:30,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:30,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:31,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:32,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:33,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:34,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:35,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:35,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:35,656 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:25:36,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:37,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:37,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:25:37,514 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:25:37,515 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:25:37,516 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:25:38,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:38,489 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:25:39,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:39,491 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:25:40,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:40,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:40,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:25:40,968 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:25:41,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:42,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:43,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:44,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:45,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:46,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:46,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:47,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:48,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:49,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:50,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:51,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:52,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:52,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:53,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:54,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:55,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:55,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:25:55,968 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:25:56,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:57,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:25:57,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:58,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:25:59,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:00,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:01,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:02,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:03,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:03,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:04,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:05,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:05,672 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:26:06,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:07,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:08,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:08,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:09,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:10,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:10,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:26:10,970 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:26:11,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:11,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:26:11,497 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:26:11,497 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:26:11,498 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:26:11,525 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:26:12,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:13,526 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:26:13,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:13,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:14,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:15,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:16,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:17,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:18,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:19,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:19,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:20,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:21,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:22,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:23,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:24,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:24,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:25,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:25,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:26:25,970 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:26:26,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:27,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:28,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:29,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:30,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:30,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:31,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:32,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:33,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:34,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:35,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:35,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:35,665 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:26:36,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:37,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:38,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:39,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:40,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:40,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:40,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:26:40,974 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:26:41,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:42,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:43,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:26:43,530 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:26:43,530 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:26:43,535 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:26:43,552 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:26:43,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:44,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:45,554 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:26:45,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:46,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:46,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:47,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:48,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:49,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:50,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:51,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:51,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:52,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:53,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:54,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:55,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:26:55,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:26:56,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:56,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:57,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:26:57,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:58,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:26:59,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:00,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:01,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:02,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:02,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:03,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:04,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:05,669 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:27:05,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:06,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:07,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:07,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:08,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:09,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:10,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:27:10,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:27:11,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:11,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:12,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:13,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:13,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:27:13,621 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:27:13,621 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:27:13,624 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:27:14,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:14,596 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:27:15,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:15,596 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:27:16,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:17,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:18,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:19,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:19,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:20,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:21,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:22,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:23,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:24,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:25,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:25,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:25,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:27:25,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:27:26,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:27,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:28,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:29,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:30,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:30,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:31,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:32,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:33,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:34,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:35,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:35,674 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:27:35,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:36,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:37,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:38,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:39,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:40,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:40,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:27:40,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:27:41,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:41,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:42,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:43,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:44,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:45,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:46,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:46,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:47,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:47,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:27:47,283 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:27:47,284 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:27:47,289 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:27:47,640 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:27:48,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:49,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:49,642 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:27:50,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:51,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:51,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:52,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:53,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:54,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:55,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:55,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:27:55,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:27:56,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:57,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:57,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:27:58,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:27:59,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:00,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:01,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:02,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:02,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:03,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:04,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:05,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:05,677 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:28:06,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:07,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:08,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:08,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:09,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:10,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:10,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:28:10,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:28:11,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:12,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:13,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:13,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:14,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:15,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:16,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:17,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:18,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:18,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:19,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:19,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:28:19,328 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:28:19,328 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:28:19,330 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:28:19,668 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:28:20,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:21,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:21,669 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:28:22,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:23,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:24,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:24,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:25,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:25,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:28:25,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:28:26,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:27,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:28,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:29,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:30,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:30,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:31,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:32,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:33,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:34,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:35,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:35,682 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:28:36,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:36,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:37,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:38,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:39,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:40,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:40,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:28:40,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:28:41,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:42,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:42,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:43,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:44,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:45,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:46,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:47,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:47,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:48,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:49,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:50,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:51,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:52,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:28:52,125 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:28:52,126 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:28:52,129 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:28:52,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:52,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:52,715 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:28:53,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:53,715 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:28:54,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:55,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:55,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:28:55,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:28:56,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:57,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:58,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:28:58,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:28:59,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:00,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:01,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:02,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:03,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:04,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:04,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:05,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:05,686 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:29:06,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:07,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:08,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:09,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:09,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:10,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:10,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:29:10,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:29:11,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:12,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:13,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:14,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:15,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:15,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:16,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:17,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:18,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:19,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:20,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:20,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:21,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:22,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:22,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:29:22,731 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:29:22,732 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:29:22,734 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:29:22,741 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:29:23,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:24,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:25,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:25,744 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:29:25,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:25,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:29:25,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:29:26,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:27,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:28,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:29,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:30,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:31,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:31,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:32,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:33,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:34,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:35,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:35,690 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:29:36,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:37,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:37,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:38,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:39,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:40,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:40,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:29:40,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:29:41,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:42,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:42,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:43,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:44,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:45,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:46,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:47,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:47,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:48,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:49,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:50,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:51,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:29:51,208 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:29:51,208 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:29:51,210 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:29:51,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:51,773 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:29:52,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:53,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:53,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:29:53,774 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:29:54,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:55,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:55,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:29:55,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:29:56,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:57,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:58,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:59,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:29:59,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:00,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:01,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:02,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:03,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:04,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:05,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:05,694 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:30:05,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:06,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:07,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:08,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:09,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:10,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:10,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:10,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:30:10,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:30:11,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:12,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:13,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:14,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:15,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:16,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:16,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:17,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:18,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:19,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:20,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:21,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:21,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:22,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:23,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:24,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:30:24,494 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:30:24,495 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:30:24,500 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:30:24,811 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:30:24,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:25,812 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:30:25,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:25,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:30:25,987 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:30:26,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:27,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:27,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:28,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:29,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:30,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:31,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:32,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:32,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:33,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:34,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:35,698 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:30:35,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:36,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:37,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:37,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:38,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:39,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:40,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:40,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:30:40,986 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:30:41,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:42,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:43,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:43,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:44,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:45,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:46,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:47,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:48,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:49,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:49,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:50,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:51,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:52,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:53,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:54,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:54,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:55,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:30:55,046 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:30:55,047 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:30:55,052 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:30:55,841 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:30:55,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:55,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:30:55,987 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:30:56,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:57,843 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:30:57,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:58,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:30:59,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:30:59,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:00,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:01,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:02,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:03,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:04,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:05,702 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:31:05,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:05,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:06,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:07,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:08,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:09,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:10,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:10,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:10,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:31:10,987 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:31:11,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:12,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:13,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:14,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:15,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:15,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:16,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:17,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:18,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:19,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:20,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:21,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:21,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:22,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:23,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:24,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:25,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:25,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:31:25,989 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:31:26,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:27,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:27,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:31:27,823 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:31:27,823 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:31:27,829 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:31:27,865 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:31:28,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:28,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:29,867 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:31:29,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:30,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:31,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:32,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:32,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:33,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:34,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:35,706 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:31:35,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:36,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:37,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:38,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:38,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:39,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:40,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:40,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:31:40,990 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:31:41,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:42,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:43,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:43,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:44,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:45,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:46,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:47,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:48,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:49,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:49,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:50,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:51,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:52,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:53,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:54,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:31:54,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:55,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:31:55,874 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:31:55,875 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:31:55,876 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:31:55,889 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:31:56,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:56,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:31:56,291 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:31:57,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:57,891 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:31:58,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:59,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:31:59,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:00,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:01,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:02,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:03,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:04,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:05,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:05,710 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:32:05,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:06,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:07,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:08,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:09,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:10,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:10,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:11,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:32:11,023 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:32:11,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:12,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:13,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:14,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:15,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:15,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:16,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:17,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:18,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:19,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:20,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:21,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:21,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:22,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:23,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:24,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:25,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:26,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:32:26,024 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:32:26,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:27,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:27,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:32:27,245 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:32:27,246 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:32:27,248 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:32:27,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:27,920 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:32:28,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:29,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:29,921 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:32:30,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:31,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:32,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:32,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:33,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:34,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:35,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:35,714 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:32:36,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:37,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:38,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:38,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:39,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:40,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:41,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:32:41,024 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:32:41,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:42,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:43,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:43,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:44,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:45,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:46,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:47,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:48,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:49,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:49,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:50,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:51,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:52,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:53,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:54,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:54,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:32:55,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:56,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:32:56,026 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:32:56,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:56,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:32:56,602 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:32:56,602 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:32:56,604 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:32:56,951 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:32:57,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:57,951 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:32:58,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:32:59,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:00,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:00,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:01,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:02,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:03,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:04,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:05,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:05,719 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:33:06,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:06,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:07,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:08,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:09,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:10,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:11,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:33:11,026 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:33:11,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:12,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:12,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:13,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:14,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:15,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:16,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:17,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:17,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:18,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:19,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:20,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:21,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:22,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:22,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:23,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:24,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:25,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:26,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:33:26,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:33:26,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:27,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:33:27,013 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:33:27,014 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:33:27,015 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:33:27,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:27,984 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:33:27,984 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:33:28,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:28,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:29,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:30,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:31,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:32,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:33,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:34,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:34,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:35,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:35,723 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:33:36,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:37,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:38,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:39,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:40,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:40,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:41,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:33:41,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:33:41,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:42,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:43,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:44,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:45,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:45,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:46,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:47,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:48,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:49,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:50,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:51,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:51,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:52,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:53,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:54,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:55,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:56,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:33:56,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:33:56,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:57,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:33:57,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:33:57,432 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:33:57,433 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:33:57,434 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:33:57,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:58,013 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:33:58,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:33:59,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:00,015 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:34:00,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:01,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:02,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:02,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:03,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:04,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:05,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:05,727 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:34:06,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:07,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:08,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:08,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:09,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:10,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:11,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:34:11,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:34:11,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:12,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:13,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:13,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:14,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:15,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:16,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:17,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:18,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:19,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:19,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:20,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:21,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:22,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:23,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:24,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:24,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:25,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:25,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:34:25,903 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:34:25,903 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:34:25,904 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:34:26,043 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:34:26,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:34:26,316 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:34:26,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:27,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:28,045 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:34:28,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:29,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:30,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:30,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:31,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:32,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:33,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:34,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:35,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:35,730 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:34:36,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:36,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:37,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:38,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:39,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:40,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:41,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:34:41,053 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:34:41,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:42,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:42,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:43,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:44,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:45,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:46,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:47,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:47,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:48,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:49,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:50,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:51,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:52,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:52,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:53,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:54,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:55,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:56,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:34:56,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:34:56,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:57,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:57,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:34:58,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:34:58,497 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:34:58,497 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:34:58,499 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:34:58,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:34:59,076 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:34:59,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:00,077 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:35:00,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:01,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:02,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:02,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:03,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:04,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:05,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:05,734 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:35:06,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:07,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:08,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:08,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:09,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:10,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:11,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:35:11,057 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:35:11,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:12,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:13,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:13,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:14,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:15,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:16,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:17,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:18,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:19,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:19,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:20,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:21,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:22,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:23,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:24,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:25,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:25,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:26,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:35:26,059 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:35:26,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:27,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:28,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:29,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:30,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:30,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:31,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:35:31,462 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:35:31,462 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:35:31,464 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:35:31,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:32,104 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:35:32,105 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:35:32,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:33,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:34,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:35,738 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:35:35,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:36,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:36,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:37,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:38,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:39,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:40,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:41,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:35:41,059 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:35:41,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:42,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:42,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:43,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:44,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:45,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:46,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:47,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:47,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:48,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:49,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:50,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:51,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:52,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:53,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:53,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:54,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:55,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:56,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:35:56,059 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:35:56,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:57,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:58,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:35:58,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:35:59,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:00,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:01,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:02,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:03,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:36:03,013 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:36:03,013 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:36:03,014 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:36:03,134 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:36:03,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:04,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:04,135 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:36:04,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:05,742 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:36:05,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:06,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:07,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:08,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:09,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:09,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:10,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:11,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:36:11,059 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:36:11,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:12,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:13,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:14,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:15,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:15,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:16,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:17,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:18,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:19,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:20,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:20,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:21,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:22,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:23,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:24,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:25,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:26,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:36:26,059 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:36:26,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:26,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:27,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:28,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:29,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:30,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:31,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:31,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:32,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:33,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:34,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:35,753 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:36:35,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:36,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:37,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:36:37,228 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:36:37,229 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:36:37,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:37,230 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:36:37,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:38,165 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:36:38,165 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:36:38,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:39,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:40,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:41,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:36:41,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:36:41,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:42,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:42,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:43,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:44,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:45,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:46,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:47,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:48,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:48,993 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:49,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:50,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:51,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:52,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:53,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:53,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:54,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:55,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:56,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:36:56,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:36:56,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:57,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:58,865 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:36:58,997 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:36:59,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:00,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:01,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:02,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:03,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:03,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:04,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:37:04,419 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:37:04,419 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:37:04,421 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:37:04,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:05,190 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:37:05,750 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:37:06,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:06,191 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:37:06,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:07,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:08,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:09,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:09,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:11,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:11,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:37:11,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:37:12,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:13,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:14,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:14,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:15,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:16,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:17,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:18,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:19,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:19,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:20,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:21,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:22,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:23,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:24,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:25,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:25,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:26,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:26,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:37:26,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:37:27,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:28,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:29,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:30,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:30,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:31,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:32,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:33,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:34,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:35,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:35,751 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:37:36,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:36,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:37:36,396 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:37:36,397 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:37:36,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:36,403 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:37:37,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:37,216 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:37:38,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:38,217 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:37:39,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:40,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:41,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:41,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:37:41,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:37:42,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:42,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:43,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:44,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:45,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:46,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:47,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:47,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:48,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:49,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:50,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:51,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:52,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:52,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:53,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:54,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:55,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:56,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:37:56,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:37:56,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:57,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:58,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:37:58,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:37:59,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:00,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:01,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:02,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:03,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:03,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:04,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:05,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:05,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:38:05,633 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:38:05,633 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:38:05,634 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:38:05,755 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:38:06,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:06,241 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:38:06,241 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:38:07,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:08,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:09,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:09,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:10,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:11,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:38:11,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:38:11,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:12,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:13,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:14,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:15,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:15,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:16,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:17,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:18,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:19,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:20,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:21,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:21,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:22,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:23,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:24,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:25,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:26,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:38:26,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:38:26,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:26,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:27,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:28,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:29,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:30,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:31,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:31,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:32,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:33,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:34,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:35,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:35,759 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:38:35,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:38:35,816 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:38:35,816 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:38:35,817 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:38:36,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:36,271 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:38:37,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:37,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:38,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:38,272 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:38:39,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:40,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:41,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:38:41,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:38:41,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:42,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:42,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:43,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:44,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:45,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:46,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:47,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:48,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:48,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:49,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:50,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:51,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:52,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:53,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:53,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:38:54,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:55,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:56,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:38:56,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:38:56,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:57,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:58,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:59,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:38:59,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:00,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:01,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:02,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:03,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:04,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:04,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:05,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:05,773 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:39:06,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:07,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:08,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:08,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:39:08,847 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:39:08,848 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:39:08,849 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:39:09,301 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:39:09,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:10,303 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:39:10,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:10,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:11,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:39:11,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:39:11,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:12,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:13,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:14,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:15,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:15,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:16,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:17,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:18,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:19,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:20,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:21,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:21,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:22,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:23,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:24,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:25,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:26,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:39:26,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:39:26,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:27,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:27,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:28,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:29,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:30,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:31,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:32,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:32,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:33,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:34,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:35,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:35,767 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:39:36,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:37,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:37,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:38,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:39,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:40,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:41,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:39:41,067 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:39:41,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:42,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:42,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:39:42,531 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:39:42,531 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:39:42,540 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:39:42,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:43,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:43,335 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:39:44,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:44,336 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:39:45,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:46,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:47,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:48,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:48,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:49,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:50,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:51,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:52,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:53,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:54,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:54,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:39:55,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:56,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:39:56,064 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:39:56,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:57,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:58,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:59,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:39:59,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:00,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:01,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:02,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:03,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:04,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:05,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:05,771 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:40:05,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:06,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:07,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:08,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:09,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:10,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:10,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:11,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:40:11,071 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:40:11,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:12,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:12,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:40:12,492 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:40:12,492 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:40:12,493 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:40:13,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:13,362 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:40:14,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:14,363 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:40:15,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:16,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:16,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:17,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:18,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:19,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:20,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:21,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:22,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:22,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:23,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:24,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:25,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:26,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:40:26,068 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:40:26,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:27,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:27,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:28,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:29,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:30,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:31,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:32,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:33,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:33,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:34,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:35,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:35,783 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:40:36,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:37,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:38,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:38,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:39,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:40,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:40,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:40:40,898 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:40:40,899 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:40:40,900 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:40:41,387 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:40:42,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:40:42,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:42,002 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:40:42,388 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:40:42,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:43,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:43,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:44,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:46,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:46,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:47,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:48,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:49,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:50,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:51,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:52,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:53,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:54,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:55,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:55,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:40:56,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:57,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:40:57,000 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:40:57,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:58,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:40:59,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:00,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:01,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:01,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:02,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:03,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:04,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:05,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:05,786 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:41:06,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:07,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:07,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:08,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:09,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:10,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:11,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:11,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:41:11,160 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:41:11,160 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:41:11,162 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:41:11,413 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:41:12,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:12,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:41:12,204 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:41:12,414 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:41:12,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:13,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:14,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:15,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:16,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:17,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:18,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:18,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:19,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:20,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:21,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:22,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:23,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:23,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:24,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:25,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:26,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:27,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:41:27,202 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:41:27,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:28,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:29,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:29,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:30,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:31,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:32,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:33,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:34,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:34,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:35,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:35,786 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:41:36,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:37,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:38,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:39,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:40,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:40,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:41,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:42,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:41:42,202 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:41:42,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:43,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:44,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:45,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:45,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:41:45,420 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:41:45,420 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:41:45,424 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:41:45,441 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:41:46,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:46,442 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:41:46,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:47,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:48,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:49,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:50,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:51,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:51,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:52,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:53,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:54,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:55,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:56,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:57,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:41:57,202 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:41:57,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:57,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:41:58,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:41:59,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:00,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:01,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:02,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:02,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:03,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:04,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:05,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:05,790 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:42:06,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:07,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:08,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:08,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:09,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:10,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:11,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:12,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:42:12,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:42:12,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:13,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:13,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:14,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:15,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:16,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:16,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:42:16,275 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:42:16,275 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:42:16,277 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:42:16,469 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:42:17,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:18,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:18,471 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:42:19,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:19,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:20,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:21,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:22,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:23,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:24,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:25,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:25,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:26,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:27,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:42:27,204 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:42:27,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:28,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:29,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:30,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:30,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:31,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:32,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:33,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:34,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:35,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:35,794 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:42:36,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:36,796 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:37,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:38,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:39,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:40,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:41,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:41,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:42,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:42:42,215 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:42:42,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:43,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:44,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:45,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:46,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:46,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:47,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:47,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:42:47,646 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:42:47,646 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:42:47,651 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:42:48,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:48,494 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:42:48,495 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:42:49,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:50,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:51,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:52,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:52,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:53,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:54,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:55,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:56,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:57,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:42:57,206 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:42:57,246 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:57,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:42:58,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:42:59,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:00,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:01,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:02,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:03,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:03,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:04,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:05,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:05,798 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:43:06,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:07,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:08,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:09,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:09,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:10,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:11,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:12,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:43:12,219 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:43:12,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:13,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:14,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:14,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:15,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:16,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:17,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:17,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:43:17,976 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:43:17,976 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:43:17,978 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:43:18,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:18,531 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:43:18,532 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:43:19,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:19,388 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:20,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:21,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:22,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:23,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:24,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:25,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:25,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:26,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:27,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:43:27,208 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:43:27,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:28,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:29,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:30,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:30,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:31,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:32,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:33,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:34,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:35,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:35,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:35,803 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:43:36,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:37,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:38,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:39,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:40,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:41,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:41,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:42,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:43:42,218 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:43:42,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:43,396 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:44,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:45,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:46,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:46,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:47,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:47,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:43:47,586 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:43:47,586 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:43:47,588 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:43:48,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:48,561 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:43:48,561 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:43:49,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:50,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:51,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:51,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:52,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:53,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:54,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:55,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:56,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:57,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:43:57,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:43:57,211 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:43:57,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:58,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:43:59,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:00,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:01,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:02,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:03,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:03,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:04,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:05,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:05,807 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:44:06,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:07,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:08,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:09,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:09,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:10,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:11,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:12,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:44:12,214 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:44:12,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:13,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:14,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:14,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:15,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:16,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:17,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:18,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:44:18,155 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:44:18,155 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:44:18,156 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:44:18,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:18,586 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:44:18,586 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:44:19,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:19,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:20,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:21,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:22,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:23,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:24,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:25,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:25,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:26,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:27,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:44:27,215 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:44:27,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:28,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:29,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:30,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:30,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:31,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:32,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:33,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:34,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:35,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:35,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:35,811 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:44:36,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:37,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:38,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:39,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:40,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:41,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:41,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:42,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:44:42,217 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:44:42,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:43,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:44,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:45,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:46,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:44:46,218 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:44:46,218 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:44:46,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:46,219 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:44:46,609 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:44:46,610 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:44:46,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:47,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:48,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:49,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:50,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:51,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:52,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:52,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:53,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:54,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:55,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:56,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:57,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:44:57,217 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:44:57,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:44:57,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:58,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:44:59,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:00,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:01,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:02,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:03,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:03,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:04,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:05,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:05,815 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:45:06,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:07,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:08,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:09,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:09,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:10,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:11,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:12,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:45:12,228 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:45:12,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:13,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:14,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:14,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:15,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:16,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:17,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:17,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:45:17,868 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:45:17,868 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:45:17,869 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:45:18,637 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:45:18,637 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:45:18,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:19,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:19,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:20,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:21,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:22,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:23,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:24,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:25,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:25,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:26,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:27,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:45:27,221 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:45:27,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:28,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:29,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:30,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:31,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:31,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:32,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:33,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:34,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:35,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:35,819 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:45:36,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:36,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:37,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:38,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:39,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:40,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:41,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:42,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:45:42,221 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:45:42,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:42,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:43,646 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:44,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:45,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:46,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:47,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:48,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:48,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:45:48,315 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:45:48,316 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:45:48,317 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:45:48,662 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:45:48,663 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:45:48,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:49,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:50,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:51,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:52,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:53,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:54,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:45:54,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:55,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:56,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:57,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:45:57,221 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:45:57,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:58,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:45:59,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:00,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:00,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:01,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:02,722 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:03,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:04,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:05,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:05,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:05,824 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:46:06,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:07,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:08,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:09,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:10,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:10,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:11,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:12,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:46:12,221 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:46:12,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:13,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:14,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:15,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:15,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:16,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:17,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:18,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:46:18,602 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:46:18,602 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:46:18,604 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:46:18,692 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:46:19,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:19,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:20,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:20,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:21,694 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:46:21,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:22,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:23,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:24,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:25,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:26,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:26,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:27,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:46:27,221 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:46:27,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:28,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:29,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:30,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:31,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:32,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:32,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:33,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:34,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:35,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:35,827 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:46:36,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:37,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:37,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:38,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:39,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:40,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:41,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:42,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:46:42,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:46:42,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:43,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:43,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:44,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:45,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:46,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:47,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:48,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:48,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:46:48,828 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:46:48,829 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:46:48,829 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:48,831 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:46:49,723 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:46:49,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:50,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:51,725 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:46:51,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:52,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:53,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:54,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:46:54,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:55,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:56,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:57,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:46:57,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:46:57,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:58,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:46:59,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:00,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:00,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:01,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:02,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:03,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:04,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:05,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:05,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:05,830 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:47:06,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:07,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:08,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:09,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:10,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:11,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:11,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:12,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:47:12,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:47:12,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:13,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:14,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:15,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:16,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:16,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:17,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:18,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:19,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:47:19,246 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:47:19,246 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:47:19,247 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:47:19,754 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:47:19,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:20,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:21,756 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:47:21,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:22,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:22,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:23,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:24,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:25,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:26,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:27,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:47:27,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:47:27,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:27,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:28,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:29,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:30,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:31,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:32,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:33,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:33,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:34,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:35,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:35,835 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:47:36,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:37,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:38,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:39,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:39,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:40,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:41,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:42,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:47:42,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:47:42,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:43,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:44,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:44,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:45,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:46,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:47,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:48,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:49,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:47:49,528 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:47:49,528 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:47:49,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:49,530 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:47:49,784 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:47:49,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:50,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:51,786 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:47:51,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:52,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:53,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:54,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:55,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:47:55,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:56,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:57,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:47:57,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:47:57,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:58,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:47:59,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:00,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:01,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:01,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:02,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:03,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:04,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:05,839 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:48:05,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:06,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:06,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:07,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:08,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:09,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:10,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:11,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:12,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:48:12,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:48:12,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:12,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:13,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:14,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:15,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:16,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:17,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:18,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:18,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:19,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:48:19,852 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:48:19,853 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:48:19,854 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:48:20,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:20,819 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:48:21,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:21,820 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:48:22,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:23,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:24,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:24,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:25,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:26,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:27,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:27,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:48:27,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:48:28,000 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:29,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:30,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:30,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:31,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:32,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:33,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:34,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:35,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:35,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:35,843 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:48:36,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:37,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:38,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:39,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:40,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:41,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:41,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:42,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:42,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:48:42,222 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:48:43,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:44,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:45,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:46,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:46,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:47,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:48,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:49,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:50,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:50,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:48:50,310 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:48:50,311 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:48:50,312 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:48:50,850 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:48:51,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:51,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:51,851 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:48:52,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:53,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:54,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:55,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:56,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:57,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:57,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:48:57,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:48:57,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:48:58,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:48:59,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:00,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:01,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:02,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:03,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:03,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:04,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:05,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:05,848 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:49:06,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:07,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:08,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:09,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:09,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:10,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:11,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:12,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:12,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:49:12,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:49:13,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:14,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:14,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:15,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:16,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:17,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:18,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:19,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:19,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:20,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:21,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:21,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:49:21,040 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:49:21,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:49:21,042 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:49:21,880 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:49:22,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:23,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:23,882 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:49:24,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:25,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:25,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:26,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:27,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:27,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:49:27,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:49:28,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:29,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:30,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:31,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:31,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:32,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:33,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:34,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:35,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:35,851 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:49:36,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:36,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:37,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:38,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:39,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:40,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:41,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:42,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:42,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:49:42,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:49:42,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:43,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:44,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:45,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:46,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:47,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:48,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:48,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:49,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:50,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:51,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:51,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:49:51,460 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:49:51,461 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:49:51,462 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:49:51,906 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:49:52,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:53,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:53,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:49:53,907 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:49:54,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:55,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:56,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:57,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:57,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:49:57,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:49:58,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:59,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:49:59,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:00,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:01,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:02,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:03,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:04,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:04,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:05,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:05,855 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:50:06,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:07,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:08,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:09,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:10,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:10,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:11,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:12,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:12,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:50:12,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:50:13,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:14,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:15,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:15,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:16,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:17,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:18,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:19,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:20,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:20,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:21,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:21,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:50:21,408 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:50:21,408 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:50:21,409 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:50:21,930 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:50:22,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:23,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:23,932 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:50:24,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:25,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:26,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:26,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:27,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:27,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:50:27,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:50:28,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:29,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:30,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:31,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:32,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:32,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:33,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:34,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:35,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:35,860 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:50:36,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:37,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:37,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:38,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:39,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:40,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:41,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:42,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:42,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:50:42,223 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:50:43,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:43,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:44,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:45,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:46,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:47,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:48,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:48,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:49,053 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:50,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:51,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:51,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:50:51,613 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:50:51,613 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:50:51,615 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:50:51,973 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:50:52,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:53,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:53,975 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:50:54,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:54,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:50:55,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:56,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:57,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:57,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:50:57,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:50:58,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:50:59,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:00,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:00,333 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:01,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:02,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:03,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:04,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:05,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:05,865 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:51:05,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:06,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:07,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:08,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:09,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:10,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:11,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:11,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:12,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:12,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:51:12,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:51:13,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:14,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:15,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:16,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:16,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:17,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:18,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:19,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:20,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:21,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:21,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:21,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:51:21,748 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:51:21,749 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:51:21,750 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:51:22,006 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:51:22,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:23,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:24,008 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:51:24,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:25,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:26,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:27,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:27,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:51:27,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:51:27,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:28,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:29,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:30,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:31,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:32,198 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:32,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:33,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:34,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:35,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:35,869 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:51:36,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:37,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:37,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:38,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:39,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:40,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:41,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:42,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:42,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:51:42,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:51:43,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:43,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:44,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:45,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:46,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:47,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:48,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:48,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:49,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:50,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:51,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:52,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:52,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:51:52,285 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:51:52,285 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:51:52,287 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:51:53,032 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:51:53,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:54,033 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:51:54,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:54,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:51:55,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:56,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:57,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:57,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:51:57,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:51:58,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:51:59,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:00,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:00,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:01,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:02,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:03,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:04,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:05,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:05,873 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:52:05,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:06,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:07,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:08,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:09,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:10,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:11,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:11,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:12,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:12,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:52:12,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:52:13,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:14,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:15,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:16,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:17,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:17,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:18,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:19,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:20,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:21,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:22,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:22,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:22,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:52:22,371 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:52:22,372 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:52:22,373 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:52:23,065 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:52:23,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:24,066 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:52:24,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:25,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:26,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:27,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:27,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:52:27,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:52:28,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:28,330 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:29,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:30,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:31,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:32,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:33,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:33,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:34,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:35,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:35,877 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:52:36,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:37,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:38,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:38,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:39,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:40,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:41,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:42,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:52:42,224 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:52:42,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:43,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:44,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:44,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:45,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:46,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:47,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:48,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:49,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:49,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:50,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:51,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:52,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:53,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:54,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:52:54,121 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:52:54,122 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:52:54,123 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:52:54,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:55,098 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:52:55,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:52:55,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:56,099 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:52:56,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:57,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:52:57,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:52:57,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:58,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:52:59,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:00,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:00,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:01,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:02,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:03,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:04,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:05,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:05,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:05,881 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:53:06,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:07,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:08,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:09,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:10,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:10,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:11,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:12,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:53:12,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:53:12,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:13,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:14,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:15,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:16,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:16,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:17,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:18,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:19,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:20,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:21,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:21,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:22,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:23,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:24,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:53:24,210 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:53:24,210 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:53:24,211 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:53:24,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:25,126 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:53:25,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:26,127 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:53:26,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:27,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:53:27,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:53:27,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:27,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:28,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:29,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:30,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:31,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:32,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:33,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:33,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:34,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:35,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:35,885 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:53:36,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:37,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:38,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:39,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:39,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:40,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:41,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:42,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:53:42,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:53:42,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:43,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:44,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:45,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:45,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:46,386 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:47,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:48,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:49,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:50,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:50,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:51,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:52,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:53,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:53,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:53:53,647 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:53:53,648 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:53:53,649 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:53:54,151 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:53:54,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:55,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:55,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:53:56,153 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:53:56,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:57,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:53:57,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:53:57,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:58,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:53:59,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:00,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:01,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:01,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:02,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:03,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:04,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:05,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:05,889 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:54:06,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:06,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:07,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:08,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:09,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:10,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:11,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:12,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:54:12,225 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:54:12,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:12,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:13,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:14,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:15,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:16,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:17,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:18,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:18,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:19,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:20,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:21,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:22,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:23,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:23,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:24,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:25,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:26,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:26,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:54:26,442 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:54:26,443 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:54:26,444 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:54:27,181 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:54:27,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:54:27,226 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:54:27,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:28,181 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:54:28,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:28,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:29,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:30,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:31,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:32,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:33,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:34,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:34,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:35,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:35,895 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:54:36,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:37,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:38,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:39,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:40,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:40,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:41,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:42,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:54:42,226 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:54:42,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:43,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:44,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:45,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:45,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:46,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:47,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:48,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:49,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:50,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:50,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:51,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:52,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:53,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:54,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:55,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:54:55,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:56,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:57,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:54:57,236 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:54:57,414 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:58,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:54:59,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:54:59,360 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:54:59,361 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:54:59,363 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:54:59,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:00,210 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:55:00,400 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:00,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:01,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:02,212 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:55:02,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:03,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:04,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:05,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:05,898 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:55:05,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:06,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:07,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:08,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:09,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:11,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:11,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:11,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:12,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:55:12,231 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:55:12,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:13,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:14,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:15,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:16,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:17,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:17,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:18,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:19,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:20,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:21,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:22,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:22,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:23,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:24,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:25,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:26,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:27,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:55:27,231 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:55:27,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:55:27,371 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:55:27,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:27,372 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:55:27,374 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:55:27,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:28,237 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:55:28,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:29,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:30,239 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:55:30,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:31,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:32,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:33,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:33,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:34,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:35,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:35,902 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:55:36,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:37,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:38,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:39,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:39,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:40,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:41,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:42,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:55:42,231 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:55:42,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:43,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:44,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:44,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:45,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:46,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:47,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:48,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:49,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:50,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:50,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:51,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:52,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:53,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:54,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:55,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:55:55,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:56,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:57,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:55:57,231 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:55:57,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:55:57,742 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:55:57,742 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:55:57,744 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:55:58,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:55:58,266 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:55:59,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:00,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:00,267 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:56:01,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:01,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:02,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:03,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:04,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:05,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:05,906 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:56:06,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:06,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:07,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:08,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:09,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:10,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:11,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:12,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:12,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:56:12,232 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:56:12,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:13,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:14,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:15,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:16,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:17,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:17,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:18,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:19,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:20,157 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:21,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:22,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:23,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:23,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:24,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:25,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:26,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:27,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:27,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:56:27,236 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:56:28,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:29,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:29,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:56:29,256 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:56:29,257 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:56:29,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:29,258 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:56:29,293 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:56:30,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:31,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:32,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:32,296 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:56:33,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:34,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:34,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:35,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:35,910 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:56:36,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:37,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:38,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:39,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:40,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:40,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:41,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:42,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:42,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:56:42,233 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:56:43,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:44,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:45,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:45,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:46,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:47,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:48,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:49,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:50,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:51,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:51,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:52,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:53,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:54,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:55,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:56,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:56,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:56:57,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:57,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:56:57,239 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:56:58,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:56:58,029 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:56:58,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:56:58,031 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:56:58,323 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:56:58,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:56:59,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:00,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:00,325 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:57:01,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:01,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:02,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:03,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:04,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:05,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:05,913 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:57:06,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:06,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:07,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:08,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:09,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:10,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:11,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:12,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:12,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:57:12,237 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:57:12,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:13,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:14,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:15,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:16,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:17,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:17,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:18,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:19,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:20,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:21,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:22,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:23,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:23,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:24,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:25,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:26,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:27,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:27,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:57:27,241 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:57:28,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:28,535 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:57:28,537 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:57:28,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:57:28,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:28,539 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:57:29,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:29,351 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:57:30,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:30,352 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:57:31,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:32,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:33,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:34,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:34,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:35,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:35,918 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:57:36,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:37,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:38,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:39,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:40,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:40,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:41,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:42,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:57:42,338 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:57:42,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:43,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:44,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:45,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:46,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:46,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:47,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:48,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:49,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:50,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:51,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:52,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:52,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:53,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:54,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:55,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:56,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:57,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:57,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:57:57,342 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:57:58,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:57:58,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:57:59,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:00,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:58:00,227 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:58:00,227 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:58:00,229 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:58:00,381 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:58:00,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:01,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:02,383 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:58:02,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:03,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:03,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:04,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:05,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:05,921 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:58:06,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:07,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:08,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:09,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:09,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:10,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:11,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:12,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:58:12,342 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:58:12,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:13,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:14,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:14,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:15,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:16,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:17,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:18,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:19,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:20,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:20,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:21,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:22,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:23,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:24,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:25,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:25,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:26,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:27,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:58:27,353 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:58:27,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:28,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:29,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:30,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:30,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:30,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:58:30,731 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:58:30,731 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:58:30,733 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:58:31,412 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:58:31,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:32,413 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:58:32,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:33,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:34,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:35,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:35,926 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:58:35,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:36,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:37,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:38,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:39,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:40,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:41,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:41,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:42,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:58:42,345 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:58:42,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:43,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:44,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:45,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:46,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:46,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:47,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:48,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:49,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:50,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:51,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:52,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:52,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:53,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:54,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:55,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:56,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:57,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:58:57,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:58:57,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:58:57,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:58,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:58:59,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:00,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:01,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:59:01,077 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:59:01,078 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:59:01,079 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:59:01,436 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:59:01,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:02,437 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:59:02,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:03,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:03,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:04,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:05,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:05,930 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:59:06,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:07,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:08,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:08,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:09,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:10,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:11,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:12,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:59:12,347 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:59:12,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:13,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:14,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:14,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:15,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:16,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:17,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:18,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:19,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:19,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:20,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:21,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:22,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:23,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:24,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:25,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:25,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:26,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:27,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:59:27,355 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:59:27,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:28,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:29,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:30,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:30,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:31,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 19:59:31,473 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 19:59:31,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 19:59:31,475 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 19:59:31,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:32,461 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 19:59:32,462 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 19:59:32,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:33,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:34,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:35,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:35,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:35,934 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 19:59:36,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:37,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:38,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:39,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:40,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:41,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:41,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:42,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:59:42,352 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:59:42,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:43,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:44,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:45,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:46,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:47,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:47,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:48,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:49,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:50,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:51,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:52,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:53,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:53,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:54,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:55,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:56,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:57,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 19:59:57,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 19:59:57,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:58,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 19:59:58,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 19:59:59,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:00,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:01,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:02,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:00:02,924 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:00:02,925 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:00:02,927 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:00:03,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:03,485 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:00:04,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:04,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:04,486 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:00:05,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:05,939 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:00:06,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:07,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:08,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:09,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:09,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:10,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:11,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:12,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:12,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:00:12,356 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:00:13,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:14,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:15,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:15,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:16,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:17,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:18,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:19,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:20,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:20,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:21,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:22,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:23,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:24,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:25,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:26,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:26,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:27,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:27,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:00:27,360 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:00:28,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:29,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:30,353 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:31,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:31,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:31,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:00:31,556 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:00:31,557 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:00:31,558 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:00:32,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:32,508 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:00:32,508 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:00:33,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:34,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:35,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:35,942 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:00:36,354 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:36,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:37,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:38,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:39,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:40,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:41,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:42,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:42,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:00:42,356 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:00:42,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:43,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:44,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:45,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:46,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:47,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:48,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:48,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:49,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:50,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:51,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:52,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:53,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:53,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:54,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:55,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:56,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:57,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:57,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:00:57,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:00:58,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:00:58,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:00:59,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:00,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:01,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:02,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:03,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:01:03,211 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:01:03,211 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:01:03,213 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:01:03,537 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:01:03,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:03,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:04,538 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:01:04,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:05,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:05,946 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:01:06,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:07,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:08,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:09,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:09,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:10,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:11,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:12,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:01:12,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:01:12,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:13,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:14,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:15,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:15,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:16,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:17,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:18,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:19,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:20,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:20,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:21,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:22,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:23,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:24,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:25,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:26,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:26,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:27,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:01:27,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:01:27,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:28,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:29,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:30,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:31,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:31,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:32,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:33,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:34,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:35,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:35,951 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:01:36,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:01:36,466 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:01:36,467 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:01:36,468 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:01:36,569 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:01:36,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:37,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:37,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:38,570 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:01:38,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:39,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:40,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:41,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:42,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:01:42,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:01:42,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:42,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:43,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:44,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:45,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:46,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:47,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:47,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:48,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:49,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:50,884 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:52,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:52,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:52,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:53,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:54,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:55,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:56,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:57,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:01:57,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:01:57,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:58,523 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:01:58,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:01:59,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:00,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:01,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:02,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:03,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:03,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:04,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:05,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:05,954 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:02:06,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:07,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:08,640 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:08,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:09,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:10,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:11,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:02:11,372 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:02:11,372 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:02:11,373 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:02:11,601 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:02:12,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:02:12,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:12,461 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:02:12,602 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:02:13,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:13,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:14,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:15,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:16,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:17,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:18,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:19,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:19,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:20,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:21,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:22,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:23,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:24,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:25,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:25,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:26,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:27,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:02:27,459 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:02:27,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:28,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:29,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:30,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:31,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:31,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:32,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:33,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:34,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:35,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:35,957 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:02:36,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:36,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:37,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:38,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:39,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:40,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:41,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:42,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:02:42,462 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:02:42,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:42,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:43,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:43,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:02:43,624 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:02:43,625 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:02:43,626 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:02:43,633 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:02:44,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:44,634 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:02:45,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:46,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:47,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:48,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:48,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:49,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:50,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:51,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:52,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:53,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:53,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:54,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:55,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:56,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:57,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:02:57,462 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:02:57,502 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:58,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:02:58,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:02:59,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:00,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:01,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:02,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:03,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:04,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:04,773 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:05,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:05,961 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:03:06,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:07,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:08,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:09,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:09,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:10,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:11,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:12,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:03:12,463 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:03:12,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:13,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:14,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:03:14,054 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:03:14,054 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:03:14,055 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:03:14,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:14,660 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:03:14,661 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:03:15,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:15,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:16,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:17,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:18,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:19,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:20,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:20,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:21,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:22,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:23,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:24,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:25,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:26,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:26,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:27,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:03:27,464 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:03:27,503 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:28,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:29,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:30,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:31,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:31,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:32,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:33,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:34,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:35,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:35,965 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:03:36,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:36,967 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:37,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:38,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:39,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:40,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:41,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:42,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:03:42,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:03:42,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:42,572 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:43,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:44,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:45,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:45,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:03:45,898 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:03:45,899 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:03:45,900 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:03:46,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:46,691 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:03:46,692 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:03:47,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:47,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:48,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:49,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:50,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:51,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:52,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:53,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:53,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:03:54,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:55,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:56,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:57,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:03:57,471 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:03:57,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:58,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:59,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:03:59,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:00,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:01,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:02,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:03,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:04,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:04,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:05,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:05,968 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:04:06,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:07,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:08,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:09,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:09,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:10,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:11,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:12,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:04:12,482 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:04:12,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:13,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:14,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:14,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:15,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:16,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:17,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:18,348 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:04:18,349 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:04:18,349 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:04:18,350 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:04:18,722 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:04:18,723 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:04:18,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:19,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:20,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:20,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:21,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:22,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:23,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:24,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:25,498 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:26,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:26,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:27,472 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:04:27,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:04:27,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:28,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:29,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:30,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:31,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:32,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:32,789 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:33,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:34,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:35,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:35,972 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:04:36,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:37,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:37,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:38,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:39,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:40,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:41,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:42,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:04:42,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:04:42,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:43,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:43,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:44,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:45,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:46,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:47,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:48,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:48,798 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:49,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:50,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:51,604 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:51,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:04:51,834 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:04:51,835 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:04:51,836 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:04:52,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:52,753 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:04:52,754 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:04:53,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:53,704 ERROR gpu :823089 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-12-26 20:04:54,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:54,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:04:55,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:56,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:57,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:04:57,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:04:57,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:58,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:04:59,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:00,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:00,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:01,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:02,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:03,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:04,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:05,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:05,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:05,976 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:05:06,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:07,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:08,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:09,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:10,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:11,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:11,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:12,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:05:12,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:05:12,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:13,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:14,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:15,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:16,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:16,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:17,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:18,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:19,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:20,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:05:20,360 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:05:20,360 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:05:20,362 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:05:20,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:20,785 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:05:20,786 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:05:21,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:22,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:22,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:23,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:24,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:25,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:26,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:27,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:05:27,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:05:27,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:27,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:28,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:29,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:30,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:32,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:32,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:33,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:34,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:35,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:35,981 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:05:36,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:37,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:37,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:38,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:39,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:40,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:41,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:42,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:42,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:05:42,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:05:43,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:43,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:44,014 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:45,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:46,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:47,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:48,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:48,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:05:48,703 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:05:48,704 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:05:48,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:48,705 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:05:48,819 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:05:49,128 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:50,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:50,821 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:05:51,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:52,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:53,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:53,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:54,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:55,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:56,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:57,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:57,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:05:57,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:05:58,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:05:58,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:05:59,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:00,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:01,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:02,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:03,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:03,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:04,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:05,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:05,984 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:06:06,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:07,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:08,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:09,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:09,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:10,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:11,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:12,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:12,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:06:12,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:06:13,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:14,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:14,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:15,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:16,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:17,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:18,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:18,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:06:18,722 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:06:18,723 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:06:18,724 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:06:18,850 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:06:19,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:20,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:20,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:20,851 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:06:21,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:22,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:23,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:24,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:25,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:25,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:26,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:27,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:27,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:06:27,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:06:28,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:29,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:30,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:31,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:31,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:32,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:33,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:34,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:35,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:35,988 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:06:36,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:36,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:37,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:38,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:39,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:40,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:41,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:42,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:42,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:06:42,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:06:42,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:43,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:44,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:45,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:46,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:47,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:47,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:48,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:49,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:50,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:50,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:06:50,793 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:06:50,793 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:06:50,794 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:06:50,876 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:06:51,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:52,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:52,877 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:06:53,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:53,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:54,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:55,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:56,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:57,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:57,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:06:57,474 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:06:58,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:06:58,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:06:59,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:00,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:01,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:02,389 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:03,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:03,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:04,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:05,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:05,993 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:07:06,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:07,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:08,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:09,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:09,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:10,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:11,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:12,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:12,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:07:12,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:07:13,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:14,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:14,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:15,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:16,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:17,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:18,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:19,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:07:19,186 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:07:19,186 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:07:19,188 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:07:19,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:19,897 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:07:20,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:20,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:20,898 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:07:21,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:22,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:23,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:24,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:25,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:25,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:26,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:27,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:27,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:07:27,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:07:28,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:29,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:30,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:31,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:31,883 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:32,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:33,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:34,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:35,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:35,997 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:07:36,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:36,999 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:37,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:38,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:39,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:40,339 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:41,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:42,340 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:42,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:07:42,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:07:42,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:43,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:44,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:45,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:46,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:47,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:47,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:48,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:48,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:07:48,651 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:07:48,651 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:07:48,652 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:07:48,923 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:07:48,924 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:07:49,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:50,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:51,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:52,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:53,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:53,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:54,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:55,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:56,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:57,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:57,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:07:57,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:07:58,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:07:58,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:07:59,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:00,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:01,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:02,344 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:03,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:04,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:04,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:05,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:06,002 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:08:06,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:07,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:08,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:09,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:10,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:10,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:11,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:12,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:08:12,475 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:08:12,569 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:13,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:14,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:15,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:16,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:16,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:17,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:18,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:19,579 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:20,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:21,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:21,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:22,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:23,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:24,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:08:24,140 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:08:24,140 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:08:24,141 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:08:24,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:24,952 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:08:25,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:25,953 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:08:26,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:26,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:27,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:08:27,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:08:27,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:28,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:29,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:30,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:31,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:32,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:32,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:33,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:34,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:35,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:36,004 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:08:36,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:37,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:38,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:38,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:39,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:40,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:41,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:42,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:08:42,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:08:42,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:43,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:43,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:44,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:45,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:46,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:47,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:48,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:49,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:49,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:50,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:51,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:52,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:53,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:08:53,206 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:08:53,207 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:08:53,208 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:08:53,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:53,980 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:08:54,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:08:54,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:55,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:55,982 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:08:56,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:57,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:08:57,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:08:57,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:58,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:08:59,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:00,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:00,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:01,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:02,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:03,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:04,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:05,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:06,008 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:09:06,014 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:06,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:07,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:08,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:09,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:10,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:11,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:11,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:12,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:09:12,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:09:12,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:13,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:14,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:15,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:16,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:16,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:17,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:18,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:19,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:20,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:21,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:21,694 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:22,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:23,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:23,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:09:23,843 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:09:23,843 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:09:23,844 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:09:24,004 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:09:24,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:25,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:26,006 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:09:26,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:26,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:27,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:09:27,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:09:27,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:28,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:29,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:30,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:31,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:32,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:32,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:33,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:34,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:35,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:36,012 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:09:36,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:37,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:38,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:38,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:39,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:40,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:41,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:42,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:09:42,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:09:42,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:43,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:43,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:44,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:45,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:46,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:47,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:48,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:49,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:49,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:50,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:51,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:51,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:09:51,766 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:09:51,766 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:09:51,768 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:09:52,039 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:09:52,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:53,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:54,040 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:09:54,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:55,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:09:55,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:56,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:57,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:09:57,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:09:57,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:58,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:09:59,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:00,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:01,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:01,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:02,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:03,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:04,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:05,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:06,016 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:10:06,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:07,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:07,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:08,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:09,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:10,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:11,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:12,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:12,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:10:12,476 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:10:12,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:13,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:14,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:15,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:16,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:17,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:17,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:18,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:19,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:20,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:21,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:10:21,051 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:10:21,051 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:10:21,053 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:10:21,068 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:10:21,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:22,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:23,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:23,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:24,071 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:10:24,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:25,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:26,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:27,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:10:27,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:10:27,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:28,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:28,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:29,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:30,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:31,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:32,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:33,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:34,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:34,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:35,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:36,020 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:10:36,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:37,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:38,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:39,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:40,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:40,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:41,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:42,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:10:42,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:10:42,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:43,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:44,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:45,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:45,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:46,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:47,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:48,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:49,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:50,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:10:50,255 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:10:50,255 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:10:50,256 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:10:50,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:50,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:51,095 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:10:51,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:52,096 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:10:52,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:53,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:54,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:55,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:56,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:10:56,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:57,476 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:10:57,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:10:57,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:58,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:10:59,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:00,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:01,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:02,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:02,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:03,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:04,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:05,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:06,024 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:11:06,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:07,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:08,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:08,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:09,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:10,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:11,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:12,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:11:12,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:11:12,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:13,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:13,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:14,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:15,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:16,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:17,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:18,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:19,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:19,341 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:11:19,341 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:11:19,342 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:11:19,343 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:11:19,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:20,121 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:11:20,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:21,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:22,123 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:11:22,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:23,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:24,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:24,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:25,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:26,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:27,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:11:27,477 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:11:27,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:28,728 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:29,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:30,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:30,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:31,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:32,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:33,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:34,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:35,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:35,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:36,029 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:11:36,729 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:37,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:38,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:39,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:40,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:41,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:41,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:43,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:44,137 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:11:44,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:11:44,473 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:11:46,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:51,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:51,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:52,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:11:57,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:11:59,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:11:59,812 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:12:00,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:01,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:02,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:06,030 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:12:07,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:08,032 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:09,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:13,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:15,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:16,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:12:16,608 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:12:17,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:19,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:24,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:24,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:26,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:27,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:30,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:35,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:36,034 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:12:38,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:38,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:12:38,627 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:12:38,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:39,492 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:12:39,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:40,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:40,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:41,493 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:12:41,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:42,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:43,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:44,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:45,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:45,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:46,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:47,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:48,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:48,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:12:48,743 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:12:49,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:50,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:50,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:51,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:52,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:53,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:54,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:55,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:56,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:12:56,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:57,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:58,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:12:59,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:00,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:01,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:01,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:02,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:03,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:03,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:13:03,743 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:13:04,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:05,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:06,039 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:13:06,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:07,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:07,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:08,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:09,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:10,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:11,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:12,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:12,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:13,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:14,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:15,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:16,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:17,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:18,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:18,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:18,743 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:13:18,743 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:13:19,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:20,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:21,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:22,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:23,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:23,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:24,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:25,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:26,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:27,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:28,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:28,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:29,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:30,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:33,731 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:36,046 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:13:39,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:44,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:49,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:51,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:13:51,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:51,808 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:13:52,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:53,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:54,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:54,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:13:55,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:56,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:57,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:58,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:13:59,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:00,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:00,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:01,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:02,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:03,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:04,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:05,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:05,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:06,047 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:14:06,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:14:06,801 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:14:06,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:07,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:08,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:09,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:10,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:10,876 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:11,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:12,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:13,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:14,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:15,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:16,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:16,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:17,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:18,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:19,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:20,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:21,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:14:21,802 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:14:21,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:21,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:22,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:23,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:24,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:25,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:26,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:27,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:27,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:28,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:29,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:30,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:31,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:32,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:33,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:33,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:34,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:35,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:36,051 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:14:36,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:14:36,801 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:14:36,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:37,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:38,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:38,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:39,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:40,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:41,805 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:42,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:43,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:44,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:44,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:45,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:46,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:47,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:48,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:49,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:50,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:50,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:51,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:14:51,802 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:14:51,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:52,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:53,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:54,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:55,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:55,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:14:56,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:57,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:58,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:14:59,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:00,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:01,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:01,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:02,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:03,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:04,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:05,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:06,055 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:15:06,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:15:06,802 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:15:06,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:06,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:07,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:08,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:09,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:10,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:11,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:12,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:12,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:13,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:14,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:15,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:16,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:17,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:17,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:18,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:19,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:20,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:21,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:15:21,802 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:15:21,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:22,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:22,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:23,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:24,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:25,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:26,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:27,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:28,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:28,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:29,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:30,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:31,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:32,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:15:33,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:36,059 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:15:39,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:44,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:49,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:54,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:15:59,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:04,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:06,061 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:16:10,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:13,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:13,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:16:13,064 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:16:14,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:15,351 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:15,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:16,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:16,244 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:16:17,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:18,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:19,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:20,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:20,246 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:16:21,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:21,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:22,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:23,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:24,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:25,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:26,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:27,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:27,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:28,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:16:28,063 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:16:28,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:29,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:30,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:31,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:32,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:32,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:33,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:34,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:35,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:36,066 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:16:36,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:37,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:38,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:38,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:39,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:40,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:41,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:42,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:43,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:43,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:16:43,068 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:16:43,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:44,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:45,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:45,750 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:16:45,752 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:16:45,752 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:16:45,753 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:16:46,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:46,270 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:16:47,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:48,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:48,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:48,272 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:16:49,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:50,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:51,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:52,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:53,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:53,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:54,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:55,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:56,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:57,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:58,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:16:58,068 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:16:58,068 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:16:58,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:16:59,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:00,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:01,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:02,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:03,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:03,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:04,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:05,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:06,070 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:17:06,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:07,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:08,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:09,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:09,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:10,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:11,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:12,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:13,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:17:13,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:17:13,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:14,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:14,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:15,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:16,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:17,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:18,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:18,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:17:18,820 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:17:18,820 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:17:18,821 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:17:19,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:19,307 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:17:19,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:20,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:20,308 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:17:21,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:22,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:23,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:24,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:25,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:25,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:26,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:27,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:28,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:17:28,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:17:28,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:29,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:30,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:30,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:31,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:32,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:33,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:34,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:35,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:36,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:36,074 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:17:36,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:37,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:38,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:39,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:40,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:41,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:42,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:42,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:43,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:17:43,094 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:17:43,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:44,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:45,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:46,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:47,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:47,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:48,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:48,801 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:17:48,803 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:17:48,803 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:17:48,805 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:17:49,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:49,340 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:17:50,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:50,341 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:17:51,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:52,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:52,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:53,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:54,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:55,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:56,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:57,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:58,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:17:58,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:17:58,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:17:58,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:17:59,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:00,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:01,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:02,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:03,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:03,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:04,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:05,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:06,078 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:18:06,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:07,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:08,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:09,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:09,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:10,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:11,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:12,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:13,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:18:13,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:18:13,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:14,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:14,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:15,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:16,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:17,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:18,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:19,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:19,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:18:19,460 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:18:19,461 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:18:19,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:19,462 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:18:20,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:20,370 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:18:20,371 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:18:21,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:22,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:23,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:24,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:25,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:25,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:26,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:27,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:28,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:18:28,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:18:28,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:29,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:30,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:30,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:31,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:32,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:33,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:34,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:35,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:36,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:36,082 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:18:36,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:37,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:38,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:39,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:40,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:41,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:42,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:42,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:43,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:18:43,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:18:43,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:44,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:45,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:46,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:47,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:47,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:48,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:49,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:50,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:51,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:52,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:53,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:53,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:18:53,438 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:18:53,439 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:18:53,440 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:18:53,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:54,400 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:18:54,401 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:18:54,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:55,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:56,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:57,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:58,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:18:58,074 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:18:58,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:18:58,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:18:59,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:00,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:01,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:02,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:03,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:04,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:04,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:05,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:06,087 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:19:06,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:07,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:08,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:09,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:09,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:10,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:11,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:12,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:13,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:19:13,074 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:19:13,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:14,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:14,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:15,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:16,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:17,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:18,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:19,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:20,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:20,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:21,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:22,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:23,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:24,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:25,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:25,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:25,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:19:25,746 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:19:25,746 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:19:25,748 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:19:26,429 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:19:26,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:27,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:28,073 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:19:28,074 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:19:28,430 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:19:28,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:29,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:30,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:30,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:31,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:32,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:33,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:34,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:35,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:36,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:36,090 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:19:36,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:37,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:38,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:39,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:40,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:41,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:42,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:42,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:43,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:19:43,074 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:19:43,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:44,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:45,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:46,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:47,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:47,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:48,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:49,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:50,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:51,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:52,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:53,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:53,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:54,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:55,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:19:55,741 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:19:55,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:19:55,742 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:19:56,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:56,459 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:19:56,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:57,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:58,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:19:58,074 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:19:58,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:19:58,461 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:19:58,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:19:59,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:00,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:01,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:02,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:03,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:04,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:04,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:05,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:06,093 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:20:06,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:07,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:08,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:09,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:09,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:10,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:11,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:12,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:13,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:20:13,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:20:13,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:14,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:14,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:15,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:16,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:17,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:18,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:19,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:20,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:20,902 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:21,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:22,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:23,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:24,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:25,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:20:25,012 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:20:25,013 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:20:25,014 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:20:25,488 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:20:25,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:26,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:26,489 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:20:26,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:27,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:28,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:20:28,074 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:20:28,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:29,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:30,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:31,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:31,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:32,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:33,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:34,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:35,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:36,098 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:20:36,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:37,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:37,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:38,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:39,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:40,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:41,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:42,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:43,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:20:43,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:20:43,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:43,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:44,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:45,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:46,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:47,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:48,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:49,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:49,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:50,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:51,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:52,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:53,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:54,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:20:54,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:55,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:20:55,711 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:20:55,711 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:20:55,712 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:20:56,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:56,517 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:20:56,518 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:20:57,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:58,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:20:58,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:20:58,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:59,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:20:59,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:00,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:01,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:02,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:03,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:04,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:05,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:05,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:06,102 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:21:06,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:07,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:08,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:09,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:10,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:11,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:11,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:12,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:13,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:21:13,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:21:13,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:14,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:15,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:16,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:16,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:17,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:18,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:19,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:20,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:21,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:22,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:22,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:23,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:24,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:25,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:26,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:26,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:21:26,455 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:21:26,456 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:21:26,457 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:21:26,549 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:21:27,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:27,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:28,074 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:21:28,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:21:28,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:28,551 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:21:29,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:30,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:31,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:32,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:33,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:33,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:34,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:35,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:36,107 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:21:36,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:37,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:38,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:39,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:39,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:40,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:41,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:42,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:43,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:21:43,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:21:43,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:44,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:44,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:45,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:46,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:47,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:48,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:49,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:50,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:50,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:51,220 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:52,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:53,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:54,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:55,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:21:55,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:55,892 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:21:55,894 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:21:55,894 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:21:55,896 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:21:56,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:56,577 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:21:57,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:58,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:21:58,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:21:58,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:21:58,579 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:21:59,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:00,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:00,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:01,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:02,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:03,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:04,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:05,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:06,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:06,111 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:22:06,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:07,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:08,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:09,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:10,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:11,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:11,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:12,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:13,075 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:22:13,075 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:22:13,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:14,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:15,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:16,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:16,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:17,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:18,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:19,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:20,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:21,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:22,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:22,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:23,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:24,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:25,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:26,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:27,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:27,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:27,645 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:22:27,646 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:22:27,646 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:22:27,647 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:22:28,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:22:28,094 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:22:28,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:28,610 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:22:28,611 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:22:29,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:30,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:31,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:32,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:32,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:33,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:34,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:35,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:36,114 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:22:36,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:37,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:38,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:38,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:39,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:40,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:41,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:42,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:43,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:22:43,093 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:22:43,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:43,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:44,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:45,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:46,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:47,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:48,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:49,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:49,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:50,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:51,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:52,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:53,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:54,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:22:54,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:55,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:56,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:57,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:58,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:22:58,095 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:22:58,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:58,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:22:58,990 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:22:58,990 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:22:58,992 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:22:59,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:22:59,635 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:23:00,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:00,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:00,636 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:23:01,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:02,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:03,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:04,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:05,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:05,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:06,119 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:23:06,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:07,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:08,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:09,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:10,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:11,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:11,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:12,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:13,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:23:13,095 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:23:13,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:14,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:15,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:16,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:16,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:17,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:18,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:19,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:20,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:21,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:22,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:22,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:23,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:24,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:25,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:26,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:27,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:27,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:28,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:23:28,100 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:23:28,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:28,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:23:28,741 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:23:28,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:23:28,742 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:23:29,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:29,663 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:23:30,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:30,664 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:23:31,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:32,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:32,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:33,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:34,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:35,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:36,123 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:23:36,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:37,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:38,115 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:38,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:39,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:40,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:41,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:42,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:43,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:23:43,100 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:23:43,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:43,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:44,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:45,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:46,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:47,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:48,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:49,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:49,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:50,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:51,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:52,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:53,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:54,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:54,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:55,562 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:56,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:57,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:58,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:23:58,106 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:23:58,563 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:59,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:23:59,203 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:23:59,203 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:23:59,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:23:59,204 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:23:59,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:23:59,688 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:24:00,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:00,689 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:24:01,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:02,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:03,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:04,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:05,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:05,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:06,127 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:24:06,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:07,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:08,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:09,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:10,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:11,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:11,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:12,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:13,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:24:13,103 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:24:13,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:14,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:15,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:16,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:16,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:17,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:18,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:19,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:20,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:21,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:22,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:22,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:23,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:24,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:25,629 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:26,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:27,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:27,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:28,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:24:28,106 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:24:28,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:28,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:24:28,837 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:24:28,837 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:24:28,839 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:24:29,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:29,717 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:24:30,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:30,718 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:24:31,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:32,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:32,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:33,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:34,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:35,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:36,132 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:24:36,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:37,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:38,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:38,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:39,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:40,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:41,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:42,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:43,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:24:43,106 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:24:43,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:43,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:44,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:45,905 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:46,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:47,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:48,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:49,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:49,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:50,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:51,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:52,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:53,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:54,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:54,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:55,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:56,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:57,912 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:58,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:24:58,115 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:24:58,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:24:59,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:24:59,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:24:59,413 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:24:59,413 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:24:59,415 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:24:59,741 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:24:59,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:00,742 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:25:00,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:01,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:02,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:03,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:04,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:05,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:05,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:06,135 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:25:06,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:07,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:08,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:09,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:10,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:11,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:11,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:12,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:13,113 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:25:13,113 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:25:13,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:14,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:15,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:16,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:16,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:17,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:18,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:19,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:20,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:21,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:22,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:22,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:23,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:24,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:25,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:26,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:27,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:27,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:28,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:25:28,120 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:25:28,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:29,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:25:29,635 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:25:29,635 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:25:29,637 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:25:29,772 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:25:30,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:30,774 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:25:31,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:32,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:32,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:33,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:34,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:35,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:36,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:36,139 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:25:37,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:38,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:38,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:39,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:40,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:41,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:42,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:43,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:43,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:25:43,117 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:25:43,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:44,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:45,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:46,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:47,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:48,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:49,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:49,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:50,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:51,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:52,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:53,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:54,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:54,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:25:55,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:56,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:57,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:58,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:58,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:25:58,117 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:25:59,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:25:59,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:00,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:01,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:02,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:03,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:04,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:04,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:26:04,297 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:26:04,298 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:26:04,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:04,299 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:26:04,801 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:26:05,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:06,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:06,144 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:26:06,803 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:26:07,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:08,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:09,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:10,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:10,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:11,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:12,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:13,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:13,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:26:13,117 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:26:14,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:15,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:15,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:16,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:17,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:18,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:19,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:20,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:21,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:21,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:22,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:23,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:24,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:25,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:26,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:26,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:27,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:28,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:28,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:26:28,117 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:26:29,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:30,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:31,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:31,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:32,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:33,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:34,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:35,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:36,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:36,146 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:26:36,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:26:36,710 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:26:36,711 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:26:36,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:36,713 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:26:36,827 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:26:37,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:38,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:38,828 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:26:39,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:40,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:41,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:42,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:42,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:43,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:26:43,117 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:26:43,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:44,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:45,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:46,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:47,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:47,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:48,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:49,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:50,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:51,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:52,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:53,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:53,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:54,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:55,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:56,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:57,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:58,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:26:58,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:26:58,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:26:58,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:26:59,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:00,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:01,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:02,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:03,134 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:04,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:04,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:04,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:27:04,378 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:27:04,378 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:27:04,379 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:27:04,853 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:27:05,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:06,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:06,152 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:27:06,854 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:27:07,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:08,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:09,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:10,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:10,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:11,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:12,135 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:13,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:27:13,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:27:13,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:14,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:15,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:15,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:16,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:17,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:18,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:19,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:20,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:21,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:21,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:22,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:23,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:24,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:25,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:26,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:27,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:27,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:28,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:27:28,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:27:28,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:29,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:30,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:31,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:32,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:32,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:33,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:34,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:35,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:35,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:27:35,684 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:27:35,685 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:27:35,687 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:27:35,881 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:27:36,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:36,155 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:27:36,882 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:27:37,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:38,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:38,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:39,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:40,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:41,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:42,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:43,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:27:43,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:27:43,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:43,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:44,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:45,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:46,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:47,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:48,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:49,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:49,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:50,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:51,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:52,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:53,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:54,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:54,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:55,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:56,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:57,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:58,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:27:58,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:27:58,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:27:59,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:27:59,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:00,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:01,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:02,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:03,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:04,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:05,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:05,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:06,163 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:28:06,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:07,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:08,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:09,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:10,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:10,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:11,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:11,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:28:11,428 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:28:11,429 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:28:11,430 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:28:11,910 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:28:12,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:12,911 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:28:13,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:28:13,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:28:13,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:14,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:15,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:15,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:16,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:17,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:18,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:19,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:20,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:21,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:21,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:22,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:23,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:24,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:25,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:26,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:27,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:27,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:28,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:28:28,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:28:28,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:29,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:30,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:31,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:32,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:32,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:33,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:34,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:35,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:36,170 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:28:36,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:37,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:38,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:38,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:39,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:40,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:41,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:41,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:28:41,574 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:28:41,575 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:28:41,576 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:28:41,938 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:28:42,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:42,939 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:28:43,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:28:43,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:28:43,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:43,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:44,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:45,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:46,469 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:47,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:48,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:49,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:49,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:50,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:51,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:52,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:53,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:54,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:55,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:28:55,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:56,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:57,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:58,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:28:58,119 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:28:58,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:28:59,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:00,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:00,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:01,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:02,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:03,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:04,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:05,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:06,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:06,172 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:29:06,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:07,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:08,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:09,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:10,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:11,153 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:11,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:12,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:13,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:29:13,125 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:29:13,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:14,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:29:14,266 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:29:14,267 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:29:14,269 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:29:14,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:14,967 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:29:14,967 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:29:15,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:16,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:16,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:17,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:18,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:19,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:20,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:21,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:22,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:22,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:23,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:24,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:25,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:26,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:27,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:28,123 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:29:28,123 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:29:28,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:28,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:29,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:30,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:31,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:32,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:33,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:34,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:34,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:35,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:36,177 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:29:36,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:37,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:38,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:39,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:39,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:40,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:41,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:42,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:42,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:29:42,852 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:29:42,853 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:29:42,855 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:29:42,995 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:29:43,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:29:43,274 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:29:43,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:44,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:44,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:44,997 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:29:45,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:46,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:47,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:48,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:49,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:50,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:50,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:51,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:52,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:53,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:54,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:55,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:29:55,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:56,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:57,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:58,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:29:58,267 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:29:58,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:29:59,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:00,331 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:00,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:01,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:02,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:03,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:04,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:05,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:06,182 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:30:06,188 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:06,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:07,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:08,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:09,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:10,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:11,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:11,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:12,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:13,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:30:13,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:30:13,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:14,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:30:14,073 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:30:14,073 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:30:14,074 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:30:14,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:15,022 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:30:15,022 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:30:15,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:16,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:17,114 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:17,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:18,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:19,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:20,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:21,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:22,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:22,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:23,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:24,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:25,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:26,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:27,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:28,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:28,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:30:28,442 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:30:28,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:29,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:30,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:31,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:32,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:33,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:33,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:34,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:35,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:36,185 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:30:36,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:37,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:38,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:39,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:39,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:40,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:41,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:42,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:43,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:30:43,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:30:43,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:44,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:44,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:45,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:46,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:46,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:30:46,789 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:30:46,789 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:30:46,791 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:30:47,046 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:30:47,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:48,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:49,048 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:30:49,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:50,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:50,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:51,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:52,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:53,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:54,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:55,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:56,355 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:30:56,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:57,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:58,357 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:30:58,358 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:30:58,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:30:59,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:00,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:01,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:01,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:02,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:03,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:04,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:05,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:06,189 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:31:06,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:07,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:07,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:08,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:09,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:10,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:11,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:12,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:12,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:13,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:31:13,362 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:31:13,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:14,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:15,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:16,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:17,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:17,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:17,865 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:31:17,866 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:31:17,866 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:31:17,867 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:31:18,073 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:31:18,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:19,074 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:31:19,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:20,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:21,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:22,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:23,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:23,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:24,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:25,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:26,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:27,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:28,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:28,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:31:28,361 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:31:28,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:29,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:30,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:31,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:32,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:33,359 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:33,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:34,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:35,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:36,192 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:31:36,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:37,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:38,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:38,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:39,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:40,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:41,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:42,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:43,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:31:43,361 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:31:43,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:43,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:44,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:45,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:46,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:47,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:31:47,513 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:31:47,513 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:31:47,515 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:31:47,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:48,102 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:31:48,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:49,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:49,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:50,104 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:31:50,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:51,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:52,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:53,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:54,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:55,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:31:55,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:56,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:57,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:58,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:31:58,361 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:31:58,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:31:59,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:00,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:00,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:01,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:02,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:03,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:04,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:05,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:06,196 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:32:06,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:06,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:07,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:08,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:09,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:10,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:11,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:11,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:12,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:13,371 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:32:13,371 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:32:13,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:14,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:15,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:16,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:16,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:17,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:18,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:19,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:32:19,202 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:32:19,202 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:32:19,203 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:32:19,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:20,128 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:32:20,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:21,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:22,130 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:32:22,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:22,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:23,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:24,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:25,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:26,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:27,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:27,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:28,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:32:28,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:32:28,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:29,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:30,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:31,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:32,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:32,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:33,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:34,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:35,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:36,201 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:32:36,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:37,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:38,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:38,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:39,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:40,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:41,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:42,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:43,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:43,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:32:43,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:32:43,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:44,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:45,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:46,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:47,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:48,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:48,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:48,979 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:32:48,980 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:32:48,981 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:32:48,982 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:32:49,160 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:32:49,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:50,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:51,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:52,162 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:32:52,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:53,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:53,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:54,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:55,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:56,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:57,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:58,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:32:58,371 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:32:58,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:32:58,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:32:59,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:00,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:01,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:02,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:03,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:04,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:04,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:05,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:06,205 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:33:06,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:07,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:08,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:09,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:10,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:10,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:11,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:12,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:13,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:33:13,377 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:33:13,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:14,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:15,505 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:15,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:16,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:17,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:18,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:19,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:33:19,453 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:33:19,454 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:33:19,455 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:33:19,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:20,194 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:33:20,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:21,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:21,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:22,196 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:33:22,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:23,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:24,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:25,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:26,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:27,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:27,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:28,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:33:28,374 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:33:28,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:29,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:30,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:31,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:32,477 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:32,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:33,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:34,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:35,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:36,209 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:33:36,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:37,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:38,211 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:38,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:39,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:40,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:41,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:42,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:43,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:43,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:33:43,374 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:33:43,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:44,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:45,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:46,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:47,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:48,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:48,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:49,960 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:50,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:33:50,060 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:33:50,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:33:50,062 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:33:50,225 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:33:50,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:51,961 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:52,226 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:33:52,961 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:53,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:53,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:54,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:55,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:56,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:57,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:58,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:33:58,374 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:33:58,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:33:58,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:33:59,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:00,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:01,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:02,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:03,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:04,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:04,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:05,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:06,213 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:34:06,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:07,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:08,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:09,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:09,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:10,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:11,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:12,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:13,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:34:13,374 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:34:13,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:14,521 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:14,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:15,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:16,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:17,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:18,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:19,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:20,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:20,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:21,806 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:34:21,807 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:34:21,807 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:34:21,809 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:34:22,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:22,250 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:34:23,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:24,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:24,251 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:34:25,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:26,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:26,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:27,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:28,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:28,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:34:28,374 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:34:29,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:30,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:31,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:31,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:32,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:33,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:34,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:35,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:36,217 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:34:36,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:37,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:37,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:38,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:39,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:40,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:41,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:42,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:42,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:43,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:43,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:34:43,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:34:44,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:45,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:46,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:47,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:47,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:48,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:49,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:50,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:51,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:51,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:34:51,434 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:34:51,434 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:34:51,436 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:34:52,279 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:34:52,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:53,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:53,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:54,281 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:34:54,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:55,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:56,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:57,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:58,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:34:58,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:34:58,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:34:58,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:34:59,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:00,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:01,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:02,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:03,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:04,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:04,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:05,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:06,221 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:35:06,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:07,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:08,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:09,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:09,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:10,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:11,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:12,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:13,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:35:13,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:35:13,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:14,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:14,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:15,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:16,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:17,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:18,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:19,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:20,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:35:20,051 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:35:20,051 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:35:20,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:20,052 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:35:20,309 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:35:20,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:21,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:22,311 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:35:22,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:23,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:24,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:25,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:25,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:26,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:27,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:28,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:35:28,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:35:28,484 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:29,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:30,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:30,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:31,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:32,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:33,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:34,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:35,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:36,226 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:35:36,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:36,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:37,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:38,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:39,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:40,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:41,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:41,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:42,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:43,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:35:43,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:35:43,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:44,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:45,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:46,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:46,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:47,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:48,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:49,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:50,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:35:50,032 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:35:50,032 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:35:50,034 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:35:50,340 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:35:50,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:51,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:52,072 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:52,342 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:35:52,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:53,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:54,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:55,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:56,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:57,389 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:35:57,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:58,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:35:58,375 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:35:58,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:35:59,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:00,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:01,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:02,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:02,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:03,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:04,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:05,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:06,229 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:36:06,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:07,492 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:08,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:08,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:09,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:10,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:11,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:12,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:13,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:36:13,382 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:36:13,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:13,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:14,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:15,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:16,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:17,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:18,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:19,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:36:19,299 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:36:19,300 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:36:19,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:19,301 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:36:19,370 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:36:19,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:20,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:21,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:22,373 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:36:22,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:23,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:24,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:24,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:25,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:26,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:27,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:28,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:36:28,379 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:36:28,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:29,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:29,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:30,804 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:31,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:32,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:33,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:34,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:35,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:35,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:36,233 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:36:36,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:37,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:38,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:39,803 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:40,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:40,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:41,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:42,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:43,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:36:43,384 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:36:43,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:44,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:45,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:45,808 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:46,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:36:46,508 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:36:46,508 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:36:46,509 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:36:46,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:47,399 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:36:47,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:48,400 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:36:48,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:49,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:50,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:51,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:51,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:52,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:53,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:54,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:55,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:56,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:57,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:36:58,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:36:58,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:36:58,383 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:36:59,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:00,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:01,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:02,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:02,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:03,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:04,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:05,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:06,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:06,237 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:37:07,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:08,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:08,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:09,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:10,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:11,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:12,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:13,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:13,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:37:13,388 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:37:13,499 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:14,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:14,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:37:14,601 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:37:14,602 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:37:14,603 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:37:15,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:15,425 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:37:16,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:16,426 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:37:17,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:18,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:19,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:19,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:20,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:21,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:22,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:23,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:24,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:25,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:25,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:26,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:27,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:28,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:28,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:37:28,388 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:37:29,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:30,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:30,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:31,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:32,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:33,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:34,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:35,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:36,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:36,242 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:37:36,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:37,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:38,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:39,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:40,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:41,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:41,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:42,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:43,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:43,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:37:43,398 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:37:44,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:45,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:46,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:46,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:47,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:48,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:48,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:37:48,682 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:37:48,682 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:37:48,684 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:37:49,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:49,459 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:37:50,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:50,460 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:37:51,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:51,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:52,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:53,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:54,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:55,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:56,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:57,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:57,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:37:58,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:37:58,391 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:37:58,392 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:37:59,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:00,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:01,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:02,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:02,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:03,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:04,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:05,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:06,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:06,247 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:38:07,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:08,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:08,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:09,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:10,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:11,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:12,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:13,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:13,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:38:13,392 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:38:13,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:14,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:15,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:16,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:17,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:17,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:38:17,796 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:38:17,796 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:38:17,798 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:38:18,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:18,486 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:38:19,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:19,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:20,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:20,487 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:38:21,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:22,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:23,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:24,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:25,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:25,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:26,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:27,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:28,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:28,392 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:38:28,392 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:38:29,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:30,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:30,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:31,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:32,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:33,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:34,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:35,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:36,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:36,250 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:38:36,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:37,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:38,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:39,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:40,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:41,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:41,418 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:42,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:43,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:43,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:38:43,395 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:38:44,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:45,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:46,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:46,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:47,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:38:47,171 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:38:47,171 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:38:47,173 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:38:47,513 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:38:47,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:48,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:48,514 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:38:49,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:50,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:51,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:52,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:52,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:53,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:54,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:55,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:56,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:57,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:58,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:38:58,395 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:38:58,395 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:38:58,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:38:59,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:00,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:01,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:02,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:03,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:04,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:04,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:05,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:06,254 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:39:06,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:07,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:08,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:09,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:10,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:10,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:11,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:12,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:13,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:13,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:39:13,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:39:14,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:15,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:15,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:16,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:17,332 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:17,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:39:17,920 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:39:17,920 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:39:17,922 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:39:18,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:18,547 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:39:19,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:20,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:20,548 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:39:21,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:21,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:22,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:23,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:24,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:25,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:26,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:27,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:27,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:28,345 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:28,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:39:28,399 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:39:29,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:30,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:31,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:32,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:32,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:33,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:34,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:35,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:36,258 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:39:36,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:37,346 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:38,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:38,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:39,347 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:40,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:41,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:42,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:43,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:43,405 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:39:43,405 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:39:43,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:44,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:45,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:45,895 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:39:45,897 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:39:45,897 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:39:45,898 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:39:46,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:46,572 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:39:47,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:48,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:48,574 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:39:49,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:49,423 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:50,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:51,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:52,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:53,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:54,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:55,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:55,423 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:39:56,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:57,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:58,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:39:58,403 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:39:58,404 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:39:59,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:00,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:00,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:01,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:02,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:03,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:04,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:05,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:06,262 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:40:06,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:06,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:07,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:08,365 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:09,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:10,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:11,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:11,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:12,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:13,372 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:13,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:40:13,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:40:14,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:15,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:15,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:40:15,801 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:40:15,802 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:40:15,803 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:40:16,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:16,602 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:40:16,602 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:40:16,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:17,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:18,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:19,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:20,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:21,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:22,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:22,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:23,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:24,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:25,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:26,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:27,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:28,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:28,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:40:28,405 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:40:28,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:29,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:30,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:31,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:32,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:33,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:34,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:34,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:35,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:36,267 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:40:36,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:37,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:38,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:39,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:39,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:40,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:41,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:42,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:43,382 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:43,407 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:40:43,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:40:44,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:44,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:45,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:46,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:47,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:48,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:49,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:40:49,064 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:40:49,064 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:40:49,066 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:40:49,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:49,631 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:40:50,105 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:50,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:50,632 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:40:51,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:52,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:53,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:54,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:55,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:40:55,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:56,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:57,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:58,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:40:58,408 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:40:58,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:40:59,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:00,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:00,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:01,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:02,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:03,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:04,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:05,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:06,271 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:41:06,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:06,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:07,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:08,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:09,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:10,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:11,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:11,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:12,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:13,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:41:13,408 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:41:13,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:14,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:15,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:16,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:16,633 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:17,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:18,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:19,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:19,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:41:19,509 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:41:19,510 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:41:19,511 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:41:19,662 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:41:20,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:20,663 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:41:21,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:22,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:22,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:23,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:24,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:25,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:26,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:27,488 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:28,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:41:28,408 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:41:28,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:28,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:29,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:30,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:31,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:32,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:33,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:34,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:34,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:35,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:36,275 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:41:36,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:37,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:38,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:39,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:40,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:40,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:41,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:42,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:43,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:41:43,408 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:41:43,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:44,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:45,490 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:45,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:46,500 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:47,497 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:48,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:49,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:50,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:51,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:51,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:52,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:41:52,238 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:41:52,239 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:41:52,240 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:41:52,688 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:41:52,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:53,700 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:54,690 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:41:54,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:55,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:56,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:57,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:41:57,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:58,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:41:58,408 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:41:58,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:41:59,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:00,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:01,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:02,515 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:02,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:03,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:04,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:05,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:06,278 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:42:06,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:07,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:08,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:08,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:09,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:10,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:11,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:12,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:13,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:42:13,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:42:13,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:13,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:14,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:15,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:16,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:17,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:18,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:19,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:19,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:20,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:21,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:22,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:23,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:42:23,426 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:42:23,427 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:42:23,428 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:42:23,719 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:42:23,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:24,720 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:42:24,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:24,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:25,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:26,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:27,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:28,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:42:28,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:42:28,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:29,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:30,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:30,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:31,850 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:32,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:33,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:34,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:35,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:36,282 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:42:36,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:36,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:37,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:38,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:39,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:40,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:41,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:41,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:42,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:43,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:42:43,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:42:43,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:44,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:45,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:46,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:47,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:47,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:48,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:49,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:50,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:51,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:52,591 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:52,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:53,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:54,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:42:54,821 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:42:54,822 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:42:54,824 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:42:55,236 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:55,748 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:42:55,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:56,749 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:42:56,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:57,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:42:57,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:58,408 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:42:58,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:42:58,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:42:59,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:00,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:01,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:02,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:03,585 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:03,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:04,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:05,970 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:06,286 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:43:06,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:07,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:08,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:08,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:09,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:10,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:11,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:12,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:13,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:43:13,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:43:13,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:13,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:14,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:15,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:16,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:17,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:18,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:19,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:19,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:20,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:21,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:22,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:23,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:24,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:43:24,281 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:43:24,281 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:43:24,283 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:43:24,775 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:43:24,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:25,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:25,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:26,777 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:43:26,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:27,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:28,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:43:28,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:43:28,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:29,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:30,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:30,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:31,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:32,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:33,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:34,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:35,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:35,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:36,291 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:43:36,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:37,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:38,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:39,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:40,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:40,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:41,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:42,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:43,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:43:43,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:43:43,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:44,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:45,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:46,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:46,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:47,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:48,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:49,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:50,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:51,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:51,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:52,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:53,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:43:53,898 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:43:53,899 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:43:53,900 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:43:54,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:54,802 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:43:54,802 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:43:55,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:56,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:56,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:43:57,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:58,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:43:58,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:43:58,409 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:43:59,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:00,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:01,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:02,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:02,592 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:03,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:04,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:05,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:06,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:06,295 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:44:07,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:08,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:08,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:09,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:10,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:11,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:12,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:13,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:13,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:44:13,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:44:13,511 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:14,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:15,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:16,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:17,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:18,049 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:18,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:19,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:20,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:21,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:22,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:23,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:23,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:24,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:24,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:44:24,072 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:44:24,072 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:44:24,073 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:44:24,832 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:44:24,832 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:44:25,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:26,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:27,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:28,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:28,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:44:28,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:44:29,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:29,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:30,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:31,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:32,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:33,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:34,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:35,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:35,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:36,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:36,299 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:44:37,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:38,096 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:39,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:40,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:40,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:41,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:42,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:43,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:43,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:44:43,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:44:44,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:45,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:46,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:46,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:47,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:48,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:49,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:50,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:51,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:51,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:52,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:53,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:54,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:54,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:44:54,262 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:44:54,262 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:44:54,263 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:44:54,866 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:44:55,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:56,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:56,867 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:44:57,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:57,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:44:58,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:44:58,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:44:58,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:44:59,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:00,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:01,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:02,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:02,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:03,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:04,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:05,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:06,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:06,303 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:45:07,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:08,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:08,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:09,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:10,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:11,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:12,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:13,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:13,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:45:13,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:45:13,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:14,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:15,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:16,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:17,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:18,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:18,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:19,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:20,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:21,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:22,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:22,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:45:22,438 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:45:22,438 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:45:22,439 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:45:22,888 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:45:23,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:24,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:24,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:24,889 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:45:25,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:26,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:27,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:28,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:28,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:45:28,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:45:29,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:30,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:30,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:31,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:32,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:33,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:34,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:35,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:35,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:36,307 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:45:36,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:37,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:38,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:39,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:40,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:40,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:41,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:42,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:43,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:43,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:45:43,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:45:44,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:45,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:45,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:46,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:47,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:48,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:49,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:50,320 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:51,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:51,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:45:51,328 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:45:51,329 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:45:51,329 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:51,330 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:45:51,925 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:45:52,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:52,925 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:45:53,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:54,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:55,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:56,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:56,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:45:57,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:58,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:45:58,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:45:58,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:45:59,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:00,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:01,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:01,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:02,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:03,323 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:04,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:05,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:06,312 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:46:06,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:07,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:07,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:08,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:09,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:10,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:11,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:12,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:12,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:13,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:46:13,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:46:13,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:14,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:15,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:16,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:17,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:17,614 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:18,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:19,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:20,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:21,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:22,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:22,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:46:22,966 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:46:22,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:46:22,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:22,967 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:46:23,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:23,951 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:46:24,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:24,952 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:46:25,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:26,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:27,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:28,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:46:28,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:46:28,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:28,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:29,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:30,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:31,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:32,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:33,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:33,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:34,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:35,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:36,316 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:46:36,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:37,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:38,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:38,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:39,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:40,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:41,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:42,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:43,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:46:43,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:46:43,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:44,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:44,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:45,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:46,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:47,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:48,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:49,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:49,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:50,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:51,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:52,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:53,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:54,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:46:54,275 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:46:54,275 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:46:54,277 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:46:54,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:54,992 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:46:54,993 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:46:55,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:46:55,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:56,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:57,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:58,410 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:46:58,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:46:58,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:46:59,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:00,615 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:00,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:01,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:02,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:03,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:04,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:05,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:06,320 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:47:06,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:06,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:07,697 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:08,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:09,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:10,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:11,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:11,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:12,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:13,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:47:13,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:47:13,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:14,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:15,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:16,616 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:16,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:17,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:18,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:19,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:20,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:21,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:21,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:22,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:23,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:24,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:25,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:26,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:27,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:47:27,125 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:47:27,125 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:47:27,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:27,127 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:47:27,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:28,023 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:47:28,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:47:28,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:47:28,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:29,024 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:47:29,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:30,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:31,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:32,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:32,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:33,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:34,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:35,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:36,324 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:47:36,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:37,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:38,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:38,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:39,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:40,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:41,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:42,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:43,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:47:43,411 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:47:43,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:43,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:44,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:45,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:46,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:47,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:48,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:48,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:49,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:50,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:51,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:52,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:53,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:53,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:54,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:55,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:56,352 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:47:56,353 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:47:56,354 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:47:56,356 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:47:56,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:57,049 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:47:57,049 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:47:57,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:58,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:47:58,412 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:47:58,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:47:59,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:47:59,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:00,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:01,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:02,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:03,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:04,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:05,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:05,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:06,328 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:48:06,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:07,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:08,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:09,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:10,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:11,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:12,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:13,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:13,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:48:13,412 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:48:14,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:15,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:15,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:16,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:17,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:18,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:19,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:20,081 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:21,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:21,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:22,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:23,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:24,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:25,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:26,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:26,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:27,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:28,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:28,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:48:28,412 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:48:28,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:48:28,826 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:48:28,827 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:48:28,828 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:48:29,076 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:48:29,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:30,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:31,078 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:48:31,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:31,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:32,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:33,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:34,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:35,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:36,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:36,332 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:48:37,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:37,334 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:38,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:39,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:40,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:41,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:42,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:42,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:43,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:43,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:48:43,412 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:48:44,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:45,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:46,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:47,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:48,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:48,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:49,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:50,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:51,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:52,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:53,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:53,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:54,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:55,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:56,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:57,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:58,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:48:58,213 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:48:58,213 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:48:58,214 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:48:58,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:48:58,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:48:58,623 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:48:58,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:48:59,106 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:48:59,106 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:48:59,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:00,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:01,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:02,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:03,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:04,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:04,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:05,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:06,336 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:49:06,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:07,360 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:08,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:09,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:10,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:10,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:11,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:12,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:13,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:13,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:49:13,622 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:49:14,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:15,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:15,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:16,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:17,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:18,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:19,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:20,363 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:21,362 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:21,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:22,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:23,366 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:24,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:25,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:26,367 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:26,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:27,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:27,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:49:27,464 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:49:27,464 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:49:27,466 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:49:28,135 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:49:28,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:28,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:49:28,622 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:49:29,138 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:49:29,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:30,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:31,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:31,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:32,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:33,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:34,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:35,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:36,340 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:49:36,368 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:37,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:37,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:38,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:39,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:40,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:41,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:42,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:42,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:43,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:43,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:49:43,622 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:49:44,369 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:45,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:46,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:47,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:47,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:48,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:49,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:50,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:51,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:52,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:53,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:53,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:54,370 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:55,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:56,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:57,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:58,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:49:58,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:49:58,630 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:49:58,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:49:59,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:00,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:00,821 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:50:00,823 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:50:00,824 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:50:00,825 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:50:01,180 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:50:01,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:02,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:03,182 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:50:03,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:04,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:04,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:05,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:06,344 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:50:06,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:07,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:08,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:09,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:10,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:10,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:11,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:12,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:13,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:13,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:50:13,627 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:50:14,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:15,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:15,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:16,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:17,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:18,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:19,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:20,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:21,377 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:21,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:22,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:23,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:24,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:25,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:26,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:26,637 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:27,385 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:28,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:50:28,268 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:50:28,269 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:50:28,270 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:50:28,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:50:28,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:28,741 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:50:29,207 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:50:29,208 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:50:29,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:30,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:31,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:31,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:32,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:33,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:34,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:35,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:36,347 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:50:36,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:37,349 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:37,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:38,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:39,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:40,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:41,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:42,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:42,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:43,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:43,735 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:50:43,736 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:50:44,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:45,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:46,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:47,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:47,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:48,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:49,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:50,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:51,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:52,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:53,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:53,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:54,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:55,425 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:56,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:57,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:58,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:58,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:50:58,738 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:50:58,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:50:59,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:50:59,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:50:59,995 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:50:59,995 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:50:59,997 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:51:00,239 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:51:00,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:01,240 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:51:01,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:02,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:03,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:04,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:04,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:05,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:06,351 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:51:06,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:07,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:08,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:09,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:10,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:10,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:11,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:12,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:13,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:13,737 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:51:13,738 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:51:14,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:15,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:15,872 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:16,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:17,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:18,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:19,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:20,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:21,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:21,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:22,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:23,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:24,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:25,437 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:26,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:26,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:27,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:28,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:28,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:51:28,742 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:51:29,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:29,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:51:29,938 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:51:29,938 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:51:29,940 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:51:30,262 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:51:30,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:31,263 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:51:31,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:31,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:32,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:33,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:34,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:35,438 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:36,356 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:51:36,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:37,358 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:37,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:38,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:39,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:40,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:41,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:42,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:42,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:43,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:43,742 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:51:43,742 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:51:44,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:45,439 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:46,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:47,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:47,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:48,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:49,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:50,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:51,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:52,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:53,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:53,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:54,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:55,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:56,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:57,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:58,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:51:58,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:51:58,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:51:58,749 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:51:59,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:00,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:00,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:52:00,836 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:52:00,837 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:52:00,838 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:52:01,291 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:52:01,292 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:52:01,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:02,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:03,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:04,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:04,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:05,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:06,358 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:52:06,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:07,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:08,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:09,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:10,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:10,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:11,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:12,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:13,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:13,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:52:13,746 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:52:14,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:15,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:15,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:16,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:17,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:18,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:19,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:20,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:21,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:21,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:22,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:23,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:24,458 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:25,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:26,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:26,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:27,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:27,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:52:27,849 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:52:27,849 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:52:27,851 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:52:28,316 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:52:28,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:28,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:52:28,746 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:52:29,317 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:52:29,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:30,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:31,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:31,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:32,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:33,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:34,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:35,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:36,362 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:52:36,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:37,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:37,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:38,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:39,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:40,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:41,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:42,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:42,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:43,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:43,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:52:43,746 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:52:44,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:45,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:46,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:47,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:47,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:48,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:49,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:50,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:51,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:52,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:53,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:53,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:54,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:55,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:56,470 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:57,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:52:57,316 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:52:57,317 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:52:57,319 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:52:57,347 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:52:57,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:58,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:52:58,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:52:58,746 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:52:58,852 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:52:59,349 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:52:59,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:00,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:01,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:02,732 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:03,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:04,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:04,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:05,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:06,366 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:53:06,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:07,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:08,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:09,733 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:10,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:10,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:11,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:12,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:13,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:13,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:53:13,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:53:14,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:15,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:15,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:16,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:17,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:18,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:19,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:20,739 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:21,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:21,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:22,740 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:23,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:24,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:25,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:53:25,427 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:53:25,428 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:53:25,430 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:53:25,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:26,378 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:53:26,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:27,379 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:53:27,468 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:27,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:28,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:53:28,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:53:28,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:29,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:30,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:31,843 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:32,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:32,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:33,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:34,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:35,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:36,370 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:53:36,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:37,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:38,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:38,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:39,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:40,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:41,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:42,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:43,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:53:43,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:53:43,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:43,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:44,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:45,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:46,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:47,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:48,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:49,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:49,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:50,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:51,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:52,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:53,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:54,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:53:54,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:55,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:56,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:53:56,279 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:53:56,279 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:53:56,280 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:53:56,407 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:53:56,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:57,410 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:53:57,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:58,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:53:58,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:53:58,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:59,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:53:59,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:00,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:01,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:02,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:03,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:04,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:05,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:05,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:06,375 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:54:06,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:07,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:08,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:09,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:10,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:10,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:11,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:12,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:13,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:54:13,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:54:13,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:14,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:15,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:15,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:16,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:17,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:18,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:19,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:20,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:21,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:21,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:22,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:23,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:24,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:25,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:26,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:27,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:54:27,435 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:54:27,436 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:54:27,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:27,438 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:54:27,439 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:54:27,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:28,439 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:54:28,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:54:28,748 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:54:28,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:29,439 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:54:29,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:30,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:31,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:32,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:32,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:33,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:34,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:35,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:36,379 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:54:36,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:37,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:38,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:38,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:39,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:40,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:41,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:42,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:43,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:54:43,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:54:43,860 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:43,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:44,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:45,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:46,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:47,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:48,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:49,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:49,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:50,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:51,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:52,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:53,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:54,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:54:54,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:55,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:56,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:54:56,020 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:54:56,021 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:54:56,022 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:54:56,462 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:54:56,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:57,463 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:54:57,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:58,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:54:58,747 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:54:58,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:59,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:54:59,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:00,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:01,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:02,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:03,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:04,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:05,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:05,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:06,382 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:55:06,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:07,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:08,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:09,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:10,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:11,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:11,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:12,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:13,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:55:13,748 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:55:13,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:14,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:15,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:16,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:16,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:17,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:18,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:19,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:20,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:21,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:22,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:22,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:23,915 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:24,920 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:25,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:26,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:27,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:27,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:28,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:55:28,757 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:55:28,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:29,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:30,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:55:30,365 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:55:30,365 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:55:30,367 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:55:30,490 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:55:30,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:31,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:32,492 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:55:32,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:33,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:33,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:34,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:35,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:36,386 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:55:36,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:37,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:38,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:38,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:39,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:40,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:41,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:42,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:43,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:55:43,753 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:55:43,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:43,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:44,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:45,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:46,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:47,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:48,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:49,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:49,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:50,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:51,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:52,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:53,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:54,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:54,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:55,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:56,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:57,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:58,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:55:58,759 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:55:58,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:55:59,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:55:59,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:00,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:56:00,669 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:56:00,669 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:56:00,671 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:56:01,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:01,521 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:56:02,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:02,521 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:56:03,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:04,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:05,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:05,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:06,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:06,390 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:56:07,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:08,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:09,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:10,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:11,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:11,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:12,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:13,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:13,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:56:13,757 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:56:14,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:15,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:16,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:16,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:17,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:18,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:19,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:20,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:21,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:22,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:22,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:23,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:24,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:25,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:26,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:27,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:27,789 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:28,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:28,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:56:28,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:56:29,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:30,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:31,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:32,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:32,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:33,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:33,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:56:33,430 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:56:33,431 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:56:33,432 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:56:33,547 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:56:34,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:35,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:36,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:36,394 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:56:36,549 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:56:37,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:38,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:38,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:39,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:40,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:41,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:42,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:43,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:43,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:56:43,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:56:43,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:44,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:45,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:46,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:47,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:48,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:49,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:49,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:50,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:51,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:52,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:53,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:54,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:55,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:55,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:56:56,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:57,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:58,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:56:58,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:56:58,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:56:59,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:00,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:00,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:01,303 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:02,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:03,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:04,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:05,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:05,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:57:05,887 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:57:05,887 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:57:05,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:05,889 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:57:06,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:06,399 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:57:06,572 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:57:07,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:08,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:08,574 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:57:09,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:10,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:11,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:11,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:12,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:13,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:13,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:57:13,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:57:14,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:15,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:16,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:16,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:17,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:18,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:19,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:20,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:21,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:22,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:22,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:23,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:24,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:25,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:26,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:27,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:28,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:28,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:57:28,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:57:28,840 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:29,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:30,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:31,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:32,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:33,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:33,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:34,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:35,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:36,242 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:57:36,244 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:57:36,244 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:57:36,246 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:57:36,402 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:57:36,601 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:57:36,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:37,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:38,602 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:57:38,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:38,844 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:39,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:40,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:41,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:42,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:43,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:43,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:57:43,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:57:43,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:44,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:45,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:46,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:47,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:48,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:49,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:49,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:50,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:51,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:52,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:53,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:54,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:54,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:57:55,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:56,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:57,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:58,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:58,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:57:58,758 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:57:59,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:57:59,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:00,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:01,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:02,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:03,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:04,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:05,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:05,853 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:06,410 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:58:06,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:07,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:08,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:58:08,087 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:58:08,087 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:58:08,088 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:58:08,630 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:58:08,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:09,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:10,631 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:58:10,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:11,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:11,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:12,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:13,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:13,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:58:13,759 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:58:14,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:15,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:16,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:16,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:17,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:18,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:19,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:20,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:21,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:22,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:22,846 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:23,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:24,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:25,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:26,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:27,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:28,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:28,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:58:28,759 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:58:28,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:29,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:30,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:31,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:32,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:33,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:34,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:34,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:35,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:36,413 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:58:36,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:37,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:38,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:39,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:58:39,024 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:58:39,024 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:58:39,025 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:58:39,656 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:58:39,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:40,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:40,657 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:58:40,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:41,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:42,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:43,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:43,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:58:43,759 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:58:44,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:45,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:45,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:46,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:47,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:48,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:49,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:50,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:51,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:51,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:52,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:53,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:54,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:55,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:56,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:56,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:58:57,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:58,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:58:58,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:58:58,759 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:58:59,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:00,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:01,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:01,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:02,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:03,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:04,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:05,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:06,417 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:59:06,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:07,419 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:07,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:08,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:09,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:10,681 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:11,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:12,682 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:12,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:13,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:59:13,509 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:59:13,509 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:59:13,511 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:59:13,694 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:59:13,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:59:13,935 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:59:14,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:14,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:15,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:16,696 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:59:16,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:17,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:18,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:18,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:19,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:20,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:21,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:22,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:23,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:23,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:24,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:25,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:26,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:27,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:28,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:28,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:59:28,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:28,930 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:59:29,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:30,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:31,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:32,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:33,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:34,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:34,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:35,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:36,421 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 20:59:36,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:37,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:38,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:39,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:39,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:40,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:41,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:42,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 20:59:42,932 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 20:59:42,932 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 20:59:42,934 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 20:59:43,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:43,726 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 20:59:43,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:59:43,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:59:44,336 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:44,727 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 20:59:45,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:45,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:46,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:47,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:48,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:49,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:50,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:50,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:51,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:52,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:53,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:54,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:55,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:56,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:56,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 20:59:57,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:58,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 20:59:58,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 20:59:58,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 20:59:59,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:00,375 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:01,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:02,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:02,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:03,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:04,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:05,376 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:06,379 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:06,430 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:00:07,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:07,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:08,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:09,378 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:10,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:11,381 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:11,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:00:11,887 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:00:11,887 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:00:11,889 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:00:12,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:12,754 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:00:12,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:13,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:13,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:00:13,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:00:14,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:14,756 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:00:15,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:16,478 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:17,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:17,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:18,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:19,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:20,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:21,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:22,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:23,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:23,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:24,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:25,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:26,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:27,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:28,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:28,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:00:28,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:00:29,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:29,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:30,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:31,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:32,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:33,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:34,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:34,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:35,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:36,434 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:00:36,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:37,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:38,491 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:39,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:39,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:40,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:41,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:00:41,327 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:00:41,327 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:00:41,328 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:00:41,785 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:00:42,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:42,786 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:00:43,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:43,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:00:43,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:00:44,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:45,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:45,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:46,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:47,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:48,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:49,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:50,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:50,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:51,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:52,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:53,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:54,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:55,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:55,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:00:56,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:57,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:58,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:00:58,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:00:58,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:00:59,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:00,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:01,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:01,300 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:02,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:03,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:04,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:05,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:06,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:06,437 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:01:06,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:07,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:08,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:09,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:10,306 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:11,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:11,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:12,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:12,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:01:12,555 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:01:12,555 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:01:12,556 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:01:12,814 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:01:13,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:13,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:01:13,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:01:14,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:14,816 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:01:15,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:16,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:17,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:17,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:18,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:19,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:20,307 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:21,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:22,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:22,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:23,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:24,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:25,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:26,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:27,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:27,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:28,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:28,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:01:28,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:01:29,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:30,308 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:31,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:32,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:33,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:33,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:34,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:35,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:36,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:36,441 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:01:37,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:38,309 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:38,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:39,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:40,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:41,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:42,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:43,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:43,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:01:43,413 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:01:43,413 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:01:43,415 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:01:43,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:43,845 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:01:43,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:01:43,930 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:01:44,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:44,846 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:01:45,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:46,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:47,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:48,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:48,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:49,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:50,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:51,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:52,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:53,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:54,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:54,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:01:55,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:56,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:57,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:58,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:01:58,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:01:58,930 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:01:59,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:00,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:00,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:01,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:02,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:03,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:04,318 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:05,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:05,943 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:06,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:06,445 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:02:07,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:08,319 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:09,326 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:10,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:10,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:11,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:12,328 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:13,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:13,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:02:13,935 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:02:14,325 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:15,067 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:02:15,068 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:02:15,068 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:02:15,069 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:02:15,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:15,879 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:02:16,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:16,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:16,880 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:02:17,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:18,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:19,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:20,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:21,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:21,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:22,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:23,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:24,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:25,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:26,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:27,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:27,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:28,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:28,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:02:28,935 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:02:29,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:30,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:31,638 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:32,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:33,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:33,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:34,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:35,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:36,449 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:02:36,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:37,639 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:38,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:38,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:39,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:40,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:41,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:42,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:43,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:43,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:02:43,937 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:02:44,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:44,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:02:44,177 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:02:44,177 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:02:44,179 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:02:44,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:44,906 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:02:45,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:46,647 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:46,908 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:02:47,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:48,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:49,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:49,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:50,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:51,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:52,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:53,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:54,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:55,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:55,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:02:56,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:57,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:58,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:02:58,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:02:58,937 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:02:59,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:00,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:01,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:01,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:02,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:03,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:04,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:05,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:06,453 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:03:06,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:06,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:07,649 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:08,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:09,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:10,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:11,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:11,953 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:12,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:13,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:13,848 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:03:13,850 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:03:13,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:03:13,851 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:03:13,936 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:03:14,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:03:14,274 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:03:14,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:15,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:16,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:16,938 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:03:17,361 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:17,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:18,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:19,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:20,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:21,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:22,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:23,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:23,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:24,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:25,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:26,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:27,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:28,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:29,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:03:29,003 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:03:29,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:29,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:30,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:31,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:32,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:33,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:34,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:35,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:35,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:36,457 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:03:36,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:37,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:38,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:39,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:40,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:40,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:41,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:42,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:43,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:44,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:03:44,003 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:03:44,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:03:44,115 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:03:44,116 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:03:44,117 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:03:44,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:44,967 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:03:45,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:45,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:46,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:46,969 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:03:47,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:48,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:49,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:50,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:51,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:51,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:52,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:53,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:54,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:55,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:56,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:03:56,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:57,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:58,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:03:59,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:03:59,003 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:03:59,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:00,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:01,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:01,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:02,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:03,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:04,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:05,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:06,460 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:04:06,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:06,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:07,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:08,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:09,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:10,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:11,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:12,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:12,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:13,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:14,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:04:14,006 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:04:14,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:14,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:04:14,991 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:04:14,991 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:04:14,993 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:04:14,995 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:04:15,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:16,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:16,997 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:04:17,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:17,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:18,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:19,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:20,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:21,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:22,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:23,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:23,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:24,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:25,668 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:26,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:27,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:28,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:28,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:29,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:04:29,006 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:04:29,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:30,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:31,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:32,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:33,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:33,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:34,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:35,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:36,464 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:04:36,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:37,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:38,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:38,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:39,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:40,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:41,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:42,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:43,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:44,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:04:44,009 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:04:44,174 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:44,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:04:44,449 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:04:44,450 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:04:44,451 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:04:44,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:45,022 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:04:45,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:46,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:47,023 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:04:47,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:48,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:49,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:50,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:50,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:51,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:52,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:53,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:54,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:55,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:56,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:04:56,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:57,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:58,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:04:59,008 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:04:59,009 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:04:59,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:00,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:01,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:01,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:02,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:03,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:04,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:05,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:06,469 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:05:06,475 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:06,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:07,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:08,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:09,911 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:10,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:11,917 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:12,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:12,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:13,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:14,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:05:14,014 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:05:14,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:15,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:16,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:05:16,185 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:05:16,185 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:05:16,187 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:05:16,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:17,051 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:05:17,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:17,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:18,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:19,053 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:05:19,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:20,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:21,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:22,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:23,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:23,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:24,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:25,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:26,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:27,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:28,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:29,011 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:05:29,011 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:05:29,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:29,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:30,923 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:31,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:32,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:33,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:34,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:35,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:35,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:36,472 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:05:36,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:37,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:38,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:39,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:40,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:41,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:41,926 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:42,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:43,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:44,016 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:05:44,016 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:05:44,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:45,925 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:46,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:46,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:05:46,534 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:05:46,534 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:05:46,536 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:05:46,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:47,082 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:05:47,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:48,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:49,083 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:05:49,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:50,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:51,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:52,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:52,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:53,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:54,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:55,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:56,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:57,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:58,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:05:58,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:05:59,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:05:59,013 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:05:59,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:00,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:01,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:02,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:03,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:03,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:04,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:05,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:06,476 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:06:06,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:07,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:08,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:08,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:09,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:10,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:11,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:12,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:13,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:14,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:06:14,013 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:06:14,103 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:14,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:15,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:16,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:17,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:18,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:19,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:20,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:20,099 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:06:20,101 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:06:20,102 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:06:20,103 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:06:20,118 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:06:20,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:21,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:22,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:23,120 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:06:23,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:24,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:25,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:25,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:26,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:27,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:28,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:29,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:06:29,013 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:06:29,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:30,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:30,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:31,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:32,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:33,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:35,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:36,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:36,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:36,481 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:06:37,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:38,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:39,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:40,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:41,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:41,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:42,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:43,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:44,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:06:44,013 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:06:44,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:45,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:46,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:46,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:47,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:48,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:49,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:50,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:51,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:52,031 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:52,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:53,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:54,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:55,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:56,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:06:56,162 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:06:56,162 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:06:56,164 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:06:56,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:57,147 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:06:57,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:06:57,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:58,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:06:59,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:06:59,014 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:06:59,148 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:06:59,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:00,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:01,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:02,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:03,106 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:03,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:04,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:05,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:06,485 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:07:06,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:07,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:08,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:08,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:09,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:10,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:11,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:12,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:13,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:14,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:07:14,014 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:07:14,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:14,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:15,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:16,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:17,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:18,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:19,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:20,030 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:20,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:21,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:22,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:23,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:24,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:25,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:26,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:26,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:27,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:28,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:29,017 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:07:29,018 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:07:29,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:30,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:31,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:31,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:31,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:07:31,819 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:07:31,819 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:07:31,820 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:07:32,178 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:07:32,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:33,179 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:07:33,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:34,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:35,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:36,490 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:07:36,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:36,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:37,593 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:38,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:39,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:40,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:41,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:42,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:42,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:43,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:44,018 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:07:44,019 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:07:44,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:45,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:46,594 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:47,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:47,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:48,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:49,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:50,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:51,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:52,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:53,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:53,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:54,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:55,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:56,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:57,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:58,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:07:58,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:07:59,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:07:59,020 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:07:59,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:00,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:01,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:08:01,296 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:08:01,297 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:08:01,298 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:08:01,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:02,208 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:08:02,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:03,209 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:08:03,337 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:03,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:04,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:05,705 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:06,493 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:08:06,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:07,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:08,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:08,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:09,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:10,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:11,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:12,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:13,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:14,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:08:14,020 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:08:14,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:14,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:15,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:16,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:17,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:18,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:19,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:20,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:20,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:21,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:22,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:23,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:24,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:25,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:25,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:26,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:27,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:28,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:29,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:08:29,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:08:29,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:30,110 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:30,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:08:30,706 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:08:30,706 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:08:30,708 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:08:31,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:31,235 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:08:31,235 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:08:31,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:32,854 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:33,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:34,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:35,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:36,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:36,497 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:08:36,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:37,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:38,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:39,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:40,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:41,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:42,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:42,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:43,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:44,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:08:44,023 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:08:44,855 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:45,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:46,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:47,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:47,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:48,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:49,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:50,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:51,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:52,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:53,037 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:53,856 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:54,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:55,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:56,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:57,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:58,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:08:58,857 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:08:59,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:08:59,028 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:08:59,861 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:00,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:01,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:02,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:03,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:03,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:04,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:09:04,045 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:09:04,045 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:09:04,047 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:09:04,264 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:09:04,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:05,265 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:09:05,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:06,501 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:09:06,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:07,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:08,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:08,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:09,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:10,862 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:11,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:12,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:13,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:14,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:09:14,025 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:09:14,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:14,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:15,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:16,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:17,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:18,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:19,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:20,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:20,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:21,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:22,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:23,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:24,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:25,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:26,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:26,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:27,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:28,866 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:29,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:09:29,025 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:09:29,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:30,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:31,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:31,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:32,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:09:32,003 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:09:32,003 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:09:32,005 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:09:32,293 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:09:32,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:33,294 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:09:33,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:34,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:35,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:36,505 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:09:36,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:36,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:37,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:38,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:39,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:40,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:41,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:42,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:42,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:43,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:44,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:09:44,025 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:09:44,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:45,868 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:46,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:47,102 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:47,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:48,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:49,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:50,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:51,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:52,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:53,046 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:53,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:54,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:55,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:56,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:57,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:58,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:09:59,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:09:59,025 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:09:59,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:09:59,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:00,952 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:01,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:02,954 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:03,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:04,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:05,055 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:05,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:10:05,279 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:10:05,279 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:10:05,281 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:10:05,329 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:10:05,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:06,509 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:10:06,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:07,331 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:10:07,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:08,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:09,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:10,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:11,051 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:11,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:12,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:13,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:14,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:10:14,025 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:10:14,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:15,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:16,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:16,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:17,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:18,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:19,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:20,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:21,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:22,052 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:22,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:23,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:24,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:25,958 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:26,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:27,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:27,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:28,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:29,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:10:29,028 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:10:29,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:30,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:31,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:32,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:32,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:10:32,930 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:10:32,930 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:10:32,932 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:10:33,350 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:33,356 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:10:33,357 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:10:34,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:35,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:36,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:36,515 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:10:37,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:37,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:38,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:39,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:40,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:41,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:42,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:43,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:43,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:44,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:10:44,028 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:10:44,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:45,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:46,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:47,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:48,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:48,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:49,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:50,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:51,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:52,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:53,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:54,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:54,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:55,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:56,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:57,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:58,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:10:59,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:10:59,045 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:10:59,156 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:10:59,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:00,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:01,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:02,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:03,175 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:04,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:04,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:11:04,447 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:11:04,447 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:11:04,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:04,448 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:11:05,172 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:05,385 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:11:05,385 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:11:06,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:06,518 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:11:07,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:08,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:09,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:10,059 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:10,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:11,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:12,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:13,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:14,028 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:11:14,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:11:14,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:15,126 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:15,176 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:16,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:17,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:18,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:19,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:20,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:21,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:21,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:22,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:23,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:24,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:25,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:26,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:27,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:27,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:28,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:29,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:11:29,033 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:11:29,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:30,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:31,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:32,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:32,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:32,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:11:32,671 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:11:32,671 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:11:32,673 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:11:33,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:33,414 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:11:33,415 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:11:34,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:35,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:36,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:36,522 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:11:37,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:37,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:38,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:39,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:40,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:41,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:42,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:43,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:43,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:44,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:11:44,029 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:11:44,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:45,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:46,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:47,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:48,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:48,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:49,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:50,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:51,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:52,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:53,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:54,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:54,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:55,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:56,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:57,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:58,200 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:11:59,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:11:59,036 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:11:59,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:11:59,197 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:00,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:01,207 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:02,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:02,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:12:02,815 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:12:02,815 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:12:02,816 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:12:03,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:03,444 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:12:03,444 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:12:04,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:05,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:05,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:06,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:06,526 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:12:07,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:08,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:09,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:10,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:11,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:11,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:12,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:13,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:14,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:12:14,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:12:14,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:15,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:16,116 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:16,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:17,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:18,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:19,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:20,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:21,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:22,070 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:22,233 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:23,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:24,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:25,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:26,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:27,071 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:27,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:28,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:29,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:12:29,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:12:29,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:30,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:31,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:32,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:32,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:33,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:34,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:35,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:35,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:12:35,447 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:12:35,448 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:12:35,450 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:12:35,470 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:12:36,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:36,530 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:12:37,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:37,472 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:12:37,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:38,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:39,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:40,238 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:41,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:42,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:43,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:43,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:44,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:12:44,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:12:44,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:45,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:46,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:47,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:48,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:48,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:49,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:50,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:51,239 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:52,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:53,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:54,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:54,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:55,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:56,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:57,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:58,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:12:59,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:12:59,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:12:59,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:12:59,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:00,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:01,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:02,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:03,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:04,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:05,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:05,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:05,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:13:05,714 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:13:05,714 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:13:05,716 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:13:06,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:06,493 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:13:06,535 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:13:07,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:07,494 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:13:08,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:09,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:10,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:11,079 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:11,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:12,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:13,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:14,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:13:14,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:13:14,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:15,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:16,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:16,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:17,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:18,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:19,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:20,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:21,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:22,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:22,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:23,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:24,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:25,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:26,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:27,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:28,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:28,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:29,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:13:29,036 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:13:29,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:30,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:31,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:32,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:33,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:33,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:34,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:35,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:35,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:13:35,294 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:13:35,294 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:13:35,296 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:13:35,522 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:13:36,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:36,537 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:13:37,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:37,524 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:13:38,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:38,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:39,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:40,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:41,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:42,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:43,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:44,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:13:44,036 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:13:44,124 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:44,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:45,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:46,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:47,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:48,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:49,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:50,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:50,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:51,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:52,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:53,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:54,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:55,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:56,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:13:56,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:57,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:58,262 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:13:59,047 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:13:59,053 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:13:59,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:00,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:01,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:01,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:02,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:03,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:04,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:04,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:14:04,561 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:14:04,562 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:14:04,564 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:14:05,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:05,546 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:14:05,547 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:14:06,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:06,541 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:14:06,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:07,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:08,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:09,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:10,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:11,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:12,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:12,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:13,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:14,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:14:14,040 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:14:14,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:15,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:16,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:17,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:17,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:18,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:19,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:20,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:21,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:22,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:23,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:23,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:24,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:25,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:26,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:27,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:28,270 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:29,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:14:29,040 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:14:29,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:29,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:30,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:31,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:32,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:33,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:34,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:35,100 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:35,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:36,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:36,543 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:14:37,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:14:37,003 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:14:37,003 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:14:37,005 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:14:37,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:37,577 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:14:37,577 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:14:38,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:39,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:40,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:40,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:41,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:42,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:43,420 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:44,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:14:44,040 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:14:44,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:45,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:45,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:46,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:47,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:48,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:49,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:50,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:51,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:51,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:52,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:53,473 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:54,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:55,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:56,098 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:14:56,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:57,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:58,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:14:59,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:14:59,040 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:14:59,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:00,474 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:01,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:01,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:02,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:03,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:04,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:05,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:06,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:06,547 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:15:06,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:06,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:15:06,929 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:15:06,929 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:15:06,930 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:15:07,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:07,605 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:15:07,605 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:15:08,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:09,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:10,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:11,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:12,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:12,480 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:13,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:14,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:15:14,040 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:15:14,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:15,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:16,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:17,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:17,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:18,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:19,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:20,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:21,481 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:22,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:23,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:23,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:24,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:25,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:26,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:27,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:28,104 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:28,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:29,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:15:29,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:15:29,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:30,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:31,482 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:32,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:33,210 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:33,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:34,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:35,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:36,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:36,559 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:15:37,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:38,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:38,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:39,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:39,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:15:39,627 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:15:39,628 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:15:39,630 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:15:39,630 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:15:40,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:41,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:41,632 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:15:42,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:43,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:43,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:44,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:15:44,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:15:44,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:45,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:46,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:47,494 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:48,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:49,107 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:49,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:50,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:51,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:52,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:53,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:54,108 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:54,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:55,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:56,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:57,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:58,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:15:59,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:15:59,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:15:59,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:15:59,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:00,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:01,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:02,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:03,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:04,501 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:05,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:05,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:06,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:06,555 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:16:07,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:08,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:09,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:09,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:16:09,898 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:16:09,899 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:16:09,902 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:16:10,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:10,657 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:16:10,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:11,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:11,658 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:16:12,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:13,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:14,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:16:14,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:16:14,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:15,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:16,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:16,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:17,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:18,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:19,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:20,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:21,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:22,111 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:22,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:23,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:24,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:25,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:26,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:27,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:27,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:28,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:29,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:16:29,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:16:29,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:30,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:31,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:32,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:32,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:33,513 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:34,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:35,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:36,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:36,563 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:16:37,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:37,564 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:38,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:16:38,025 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:16:38,026 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:16:38,027 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:16:38,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:38,686 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:16:39,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:39,687 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:16:40,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:41,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:42,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:43,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:43,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:44,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:16:44,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:16:44,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:45,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:46,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:47,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:48,133 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:48,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:49,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:50,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:51,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:52,580 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:53,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:54,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:54,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:55,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:56,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:57,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:58,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:16:59,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:16:59,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:16:59,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:16:59,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:00,581 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:01,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:02,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:03,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:04,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:05,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:05,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:06,561 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:17:06,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:07,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:17:07,007 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:17:07,008 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:17:07,010 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:17:07,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:07,720 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:17:07,720 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:17:08,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:09,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:10,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:11,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:11,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:12,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:13,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:14,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:17:14,041 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:17:14,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:15,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:16,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:16,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:17,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:18,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:19,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:20,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:21,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:22,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:22,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:23,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:24,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:25,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:26,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:27,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:27,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:28,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:29,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:17:29,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:17:29,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:30,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:31,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:32,142 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:32,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:33,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:34,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:35,596 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:36,570 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:17:36,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:37,504 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:17:37,506 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:17:37,507 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:17:37,507 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:37,508 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:17:37,745 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:17:37,745 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:17:37,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:38,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:39,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:40,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:41,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:42,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:43,127 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:43,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:44,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:17:44,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:17:44,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:45,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:46,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:47,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:48,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:48,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:49,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:50,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:51,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:52,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:53,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:54,128 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:54,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:55,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:56,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:57,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:58,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:17:59,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:17:59,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:17:59,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:17:59,931 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:00,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:01,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:02,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:03,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:04,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:05,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:05,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:06,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:18:06,062 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:18:06,062 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:18:06,063 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:18:06,571 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:18:06,776 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:18:06,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:07,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:08,777 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:18:08,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:09,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:10,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:11,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:11,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:12,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:13,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:14,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:18:14,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:18:14,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:15,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:16,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:16,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:17,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:18,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:19,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:20,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:21,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:22,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:22,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:23,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:24,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:25,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:26,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:27,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:28,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:28,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:29,041 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:18:29,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:18:29,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:30,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:31,949 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:32,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:33,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:33,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:34,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:35,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:18:35,784 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:18:35,784 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:18:35,786 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:18:35,805 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:18:36,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:36,576 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:18:37,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:38,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:38,578 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:38,807 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:18:39,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:40,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:41,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:42,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:43,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:44,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:18:44,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:18:44,140 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:44,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:45,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:46,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:47,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:48,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:49,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:50,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:50,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:51,202 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:52,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:53,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:54,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:55,132 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:18:55,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:56,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:57,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:58,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:18:59,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:18:59,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:18:59,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:00,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:00,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:01,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:02,214 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:03,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:04,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:05,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:06,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:06,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:06,579 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:19:07,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:19:07,113 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:19:07,114 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:19:07,116 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:19:07,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:07,831 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:19:08,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:08,832 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:19:09,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:10,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:11,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:11,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:12,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:13,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:14,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:19:14,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:19:14,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:15,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:16,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:17,137 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:17,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:18,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:19,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:20,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:21,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:22,209 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:22,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:23,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:24,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:25,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:26,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:27,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:28,208 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:28,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:29,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:19:29,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:19:29,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:30,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:31,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:32,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:33,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:33,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:34,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:35,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:36,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:36,581 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:19:36,680 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:19:36,682 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:19:36,683 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:19:36,684 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:19:36,868 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:19:37,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:38,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:38,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:38,870 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:19:39,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:40,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:41,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:42,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:43,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:44,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:19:44,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:19:44,118 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:44,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:45,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:46,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:47,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:48,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:49,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:49,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:50,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:51,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:52,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:53,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:54,213 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:19:54,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:55,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:56,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:57,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:58,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:19:59,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:19:59,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:19:59,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:00,212 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:00,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:01,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:02,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:03,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:04,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:05,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:05,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:06,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:06,590 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:20:07,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:07,741 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:20:07,742 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:20:07,743 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:20:07,744 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:20:07,905 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:20:08,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:08,906 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:20:09,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:10,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:11,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:11,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:12,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:13,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:14,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:20:14,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:20:14,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:15,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:16,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:17,215 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:17,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:18,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:19,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:20,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:21,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:22,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:22,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:23,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:24,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:25,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:26,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:27,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:27,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:28,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:29,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:20:29,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:20:29,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:30,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:31,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:32,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:33,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:33,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:34,299 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:35,297 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:36,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:36,598 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:20:37,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:38,301 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:38,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:39,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:39,566 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:20:39,568 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:20:39,568 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:20:39,570 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:20:39,937 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:20:40,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:40,938 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:20:41,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:42,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:43,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:43,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:44,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:20:44,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:20:44,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:45,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:46,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:47,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:48,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:49,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:49,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:50,302 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:51,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:52,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:53,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:54,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:20:54,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:55,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:56,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:57,304 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:58,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:20:59,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:20:59,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:20:59,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:00,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:00,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:01,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:02,305 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:03,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:04,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:05,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:05,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:06,314 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:06,594 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:21:07,315 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:08,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:08,790 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:21:08,792 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:21:08,792 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:21:08,797 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:21:08,965 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:21:09,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:10,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:10,967 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:21:11,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:11,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:12,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:13,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:14,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:21:14,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:21:14,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:15,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:16,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:17,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:17,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:18,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:19,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:20,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:21,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:22,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:23,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:23,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:24,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:25,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:26,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:27,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:28,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:29,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:21:29,043 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:21:29,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:29,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:30,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:31,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:32,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:33,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:34,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:34,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:35,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:36,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:36,604 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:21:37,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:38,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:39,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:39,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:40,335 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:21:40,336 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:21:40,337 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:21:40,342 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:21:40,990 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:21:41,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:41,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:42,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:42,992 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:21:43,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:44,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:21:44,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:21:44,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:44,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:45,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:46,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:47,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:48,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:49,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:50,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:50,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:51,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:52,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:53,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:54,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:55,263 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:21:55,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:56,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:57,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:58,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:21:59,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:21:59,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:21:59,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:00,670 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:01,261 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:01,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:02,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:03,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:04,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:05,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:06,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:06,601 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:22:06,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:07,676 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:08,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:09,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:10,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:11,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:11,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:12,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:22:12,237 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:22:12,237 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:22:12,242 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:22:12,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:13,020 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:22:13,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:14,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:22:14,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:22:14,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:15,021 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:22:15,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:16,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:17,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:17,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:18,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:19,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:20,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:21,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:22,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:22,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:23,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:24,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:25,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:26,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:27,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:28,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:28,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:29,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:22:29,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:22:29,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:30,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:31,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:32,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:33,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:33,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:34,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:35,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:36,602 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:22:36,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:37,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:38,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:38,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:39,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:40,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:41,688 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:42,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:43,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:22:43,009 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:22:43,009 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:22:43,012 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:22:43,045 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:22:43,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:44,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:22:44,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:22:44,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:44,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:45,051 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:22:45,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:46,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:47,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:48,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:49,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:49,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:50,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:51,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:52,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:53,689 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:54,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:55,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:22:55,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:56,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:57,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:58,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:22:59,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:22:59,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:22:59,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:00,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:00,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:01,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:02,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:03,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:04,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:05,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:06,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:06,606 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:23:06,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:07,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:08,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:09,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:10,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:11,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:11,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:12,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:23:12,219 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:23:12,220 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:23:12,221 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:23:12,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:13,082 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:23:13,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:14,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:23:14,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:23:14,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:15,084 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:23:15,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:16,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:17,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:17,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:18,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:19,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:20,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:21,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:22,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:22,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:23,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:24,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:25,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:26,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:27,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:27,767 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:28,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:29,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:23:29,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:23:29,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:30,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:31,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:32,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:33,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:33,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:34,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:35,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:36,614 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:23:36,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:37,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:38,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:38,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:39,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:40,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:41,642 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:23:41,643 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:23:41,644 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:23:41,645 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:23:42,116 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:23:42,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:43,118 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:23:43,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:43,684 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:44,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:23:44,044 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:23:44,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:45,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:46,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:47,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:48,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:49,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:49,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:50,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:51,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:52,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:53,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:54,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:55,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:23:55,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:56,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:57,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:58,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:23:59,044 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:23:59,045 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:23:59,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:00,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:00,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:01,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:02,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:03,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:04,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:05,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:05,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:06,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:06,619 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:24:07,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:08,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:09,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:10,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:10,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:11,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:12,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:13,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:14,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:24:14,045 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:24:14,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:14,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:24:14,801 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:24:14,801 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:24:14,803 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:24:15,146 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:24:15,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:15,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:16,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:17,148 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:24:17,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:18,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:19,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:20,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:21,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:21,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:22,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:23,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:24,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:25,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:26,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:27,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:27,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:28,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:29,045 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:24:29,045 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:24:29,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:30,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:31,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:32,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:32,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:33,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:34,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:35,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:36,622 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:24:36,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:37,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:37,662 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:38,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:39,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:40,663 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:41,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:42,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:43,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:43,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:44,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:24:44,053 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:24:44,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:45,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:46,667 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:47,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:24:47,466 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:24:47,467 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:24:47,468 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:24:47,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:48,178 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:24:48,506 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:48,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:49,180 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:24:49,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:50,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:51,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:52,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:53,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:54,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:24:54,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:55,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:56,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:57,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:58,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:24:59,050 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:24:59,050 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:24:59,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:00,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:00,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:01,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:02,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:03,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:04,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:05,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:05,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:06,625 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:25:06,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:07,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:08,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:09,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:10,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:11,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:11,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:12,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:13,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:14,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:25:14,058 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:25:14,885 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:15,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:16,294 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:16,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:17,069 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:25:17,070 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:25:17,071 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:25:17,072 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:25:17,209 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:25:17,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:18,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:19,238 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:25:19,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:20,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:21,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:22,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:22,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:23,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:24,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:25,886 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:26,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:27,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:28,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:28,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:29,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:25:29,054 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:25:29,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:30,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:31,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:32,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:33,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:33,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:34,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:35,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:36,629 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:25:36,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:37,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:38,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:38,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:39,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:40,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:41,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:42,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:43,893 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:44,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:25:44,065 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:25:44,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:44,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:45,897 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:46,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:47,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:48,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:25:48,192 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:25:48,193 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:25:48,195 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:25:48,268 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:25:48,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:49,232 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:49,268 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:25:49,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:50,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:51,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:52,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:53,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:54,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:25:54,898 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:55,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:56,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:57,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:58,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:25:59,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:25:59,058 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:25:59,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:00,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:00,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:01,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:02,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:03,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:04,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:05,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:05,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:06,633 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:26:06,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:07,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:08,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:09,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:10,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:11,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:11,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:12,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:13,910 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:14,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:26:14,058 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:26:14,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:15,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:16,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:16,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:17,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:18,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:19,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:26:19,133 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:26:19,133 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:26:19,135 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:26:19,301 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:26:19,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:20,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:21,303 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:26:21,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:21,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:22,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:23,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:24,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:25,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:26,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:27,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:27,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:28,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:29,058 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:26:29,058 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:26:29,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:30,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:32,006 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:33,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:33,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:34,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:35,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:36,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:36,638 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:26:37,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:38,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:38,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:39,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:40,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:41,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:42,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:43,005 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:44,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:44,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:26:44,069 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:26:44,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:45,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:46,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:47,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:48,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:49,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:49,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:49,849 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:26:49,850 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:26:49,850 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:26:49,852 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:26:50,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:50,328 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:26:51,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:51,329 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:26:52,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:53,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:54,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:55,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:55,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:26:56,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:57,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:58,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:26:59,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:26:59,060 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:26:59,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:00,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:01,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:01,532 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:02,268 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:03,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:04,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:05,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:06,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:06,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:06,642 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:27:07,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:08,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:09,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:10,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:11,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:12,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:12,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:13,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:14,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:27:14,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:27:14,269 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:15,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:16,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:17,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:17,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:18,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:19,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:20,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:20,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:27:20,376 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:27:20,376 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:27:20,377 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:27:21,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:21,359 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:27:21,359 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:27:22,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:23,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:23,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:24,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:25,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:26,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:27,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:28,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:28,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:29,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:27:29,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:27:29,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:30,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:31,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:32,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:33,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:34,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:34,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:35,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:36,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:36,646 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:27:37,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:38,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:39,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:40,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:40,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:41,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:42,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:43,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:44,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:27:44,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:27:44,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:45,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:45,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:46,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:47,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:48,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:49,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:50,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:50,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:27:50,530 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:27:50,530 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:27:50,531 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:27:50,570 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:51,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:51,384 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:27:52,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:53,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:53,386 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:27:54,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:55,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:56,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:56,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:27:57,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:58,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:27:59,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:27:59,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:27:59,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:00,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:01,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:02,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:02,541 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:03,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:04,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:05,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:06,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:06,650 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:28:07,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:07,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:08,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:09,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:10,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:11,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:12,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:13,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:13,605 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:14,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:28:14,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:28:14,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:15,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:16,286 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:17,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:18,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:18,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:19,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:20,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:21,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:22,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:23,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:28:23,014 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:28:23,015 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:28:23,016 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:28:23,419 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:28:23,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:24,054 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:24,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:25,421 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:28:25,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:26,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:27,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:28,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:29,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:28:29,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:28:29,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:29,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:30,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:31,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:32,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:33,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:34,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:34,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:35,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:36,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:36,654 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:28:37,457 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:38,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:39,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:39,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:40,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:41,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:42,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:43,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:44,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:28:44,061 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:28:44,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:45,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:45,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:46,508 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:47,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:48,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:49,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:50,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:51,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:51,606 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:52,509 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:53,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:54,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:55,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:56,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:56,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:28:57,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:58,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:28:58,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:28:58,985 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:28:58,985 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:28:58,987 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:28:59,394 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:28:59,395 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:28:59,459 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:28:59,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:00,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:01,460 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:29:01,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:02,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:02,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:03,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:04,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:05,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:06,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:06,658 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:29:07,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:07,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:08,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:09,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:10,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:11,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:12,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:13,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:13,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:14,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:29:14,136 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:29:14,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:15,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:16,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:17,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:18,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:18,613 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:19,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:20,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:21,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:22,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:23,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:24,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:24,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:25,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:26,528 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:27,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:28,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:29,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:29:29,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:29:29,534 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:29,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:30,530 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:31,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:29:31,388 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:29:31,389 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:29:31,390 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:29:31,494 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:29:31,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:32,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:33,495 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:29:33,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:34,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:35,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:35,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:36,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:36,663 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:29:37,537 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:38,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:39,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:40,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:41,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:41,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:42,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:43,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:44,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:29:44,139 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:29:44,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:45,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:46,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:46,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:47,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:48,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:49,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:50,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:51,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:51,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:52,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:53,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:54,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:55,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:56,539 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:57,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:57,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:29:58,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:29:59,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:29:59,139 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:29:59,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:00,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:01,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:02,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:03,194 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:30:03,195 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:30:03,196 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:30:03,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:03,197 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:30:03,527 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:30:03,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:04,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:05,528 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:30:05,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:06,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:06,666 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:30:07,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:08,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:08,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:09,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:10,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:11,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:12,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:13,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:14,139 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:30:14,139 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:30:14,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:14,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:15,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:16,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:17,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:18,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:19,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:19,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:20,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:21,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:22,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:23,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:24,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:25,610 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:25,621 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:26,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:27,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:28,612 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:29,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:30:29,145 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:30:29,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:30,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:30,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:31,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:32,039 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:30:32,041 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:30:32,042 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:30:32,043 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:30:32,556 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:30:32,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:33,557 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:30:33,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:34,617 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:35,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:36,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:36,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:36,670 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:30:37,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:38,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:39,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:40,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:41,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:42,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:42,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:43,618 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:44,141 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:30:44,142 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:30:44,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:45,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:46,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:47,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:48,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:48,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:49,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:50,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:51,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:52,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:53,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:53,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:54,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:55,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:56,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:57,620 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:58,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:30:58,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:30:59,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:30:59,146 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:30:59,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:00,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:01,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:02,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:03,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:03,634 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:04,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:31:04,197 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:31:04,197 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:31:04,198 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:31:04,584 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:31:04,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:05,585 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:31:05,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:06,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:06,674 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:31:07,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:08,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:08,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:09,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:10,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:11,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:12,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:13,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:14,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:31:14,144 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:31:14,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:14,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:15,628 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:16,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:17,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:18,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:19,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:19,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:20,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:21,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:22,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:23,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:24,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:25,627 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:25,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:26,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:27,652 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:28,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:29,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:31:29,148 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:31:29,653 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:30,635 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:30,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:31,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:32,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:33,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:34,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:31:34,093 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:31:34,093 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:31:34,094 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:31:34,611 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:31:34,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:35,612 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:31:35,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:36,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:36,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:36,678 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:31:37,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:38,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:39,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:40,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:41,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:42,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:42,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:43,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:44,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:31:44,149 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:31:44,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:45,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:46,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:47,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:48,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:48,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:49,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:50,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:51,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:52,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:53,631 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:53,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:54,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:55,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:56,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:57,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:58,636 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:31:58,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:31:59,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:31:59,152 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:31:59,659 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:00,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:01,660 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:02,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:03,644 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:03,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:04,517 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:32:04,519 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:32:04,519 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:32:04,520 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:32:04,641 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:32:04,936 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:05,642 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:32:05,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:06,683 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:32:06,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:07,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:08,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:08,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:09,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:10,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:11,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:12,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:13,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:14,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:32:14,153 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:32:14,292 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:14,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:15,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:16,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:17,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:18,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:19,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:19,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:20,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:21,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:22,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:23,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:24,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:25,669 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:25,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:26,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:27,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:28,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:29,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:32:29,155 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:32:29,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:30,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:30,944 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:31,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:32,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:33,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:32:33,764 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:32:33,765 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:32:33,766 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:32:34,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:34,667 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:32:35,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:35,668 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:32:36,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:36,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:36,687 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:32:37,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:38,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:39,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:40,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:41,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:42,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:42,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:43,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:44,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:32:44,155 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:32:44,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:45,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:46,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:47,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:48,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:48,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:49,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:50,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:51,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:52,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:53,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:53,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:54,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:55,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:56,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:57,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:58,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:32:58,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:32:59,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:32:59,159 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:32:59,199 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:00,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:01,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:02,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:03,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:33:03,120 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:33:03,121 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:33:03,122 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:33:03,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:03,699 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:33:04,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:04,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:05,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:05,701 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:33:06,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:06,691 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:33:07,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:08,546 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:09,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:09,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:10,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:11,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:12,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:13,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:14,155 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:33:14,155 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:33:14,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:14,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:15,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:16,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:17,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:18,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:19,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:19,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:20,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:21,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:22,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:23,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:24,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:25,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:25,678 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:26,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:27,548 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:28,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:29,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:33:29,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:33:29,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:30,549 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:30,687 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:31,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:32,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:32,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:33:32,848 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:33:32,848 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:33:32,849 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:33:33,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:33,732 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:33:33,732 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:33:34,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:35,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:36,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:36,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:36,695 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:33:37,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:38,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:39,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:40,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:41,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:42,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:42,683 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:43,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:44,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:33:44,162 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:33:44,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:45,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:46,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:47,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:47,749 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:48,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:49,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:50,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:51,650 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:52,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:53,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:53,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:54,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:55,651 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:56,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:57,654 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:58,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:33:58,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:33:59,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:33:59,167 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:33:59,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:00,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:01,655 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:02,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:34:02,277 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:34:02,277 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:34:02,278 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:34:02,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:02,764 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:34:03,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:03,765 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:34:04,317 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:04,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:05,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:06,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:06,700 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:34:07,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:08,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:09,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:09,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:10,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:11,695 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:12,696 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:13,877 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:14,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:34:14,164 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:34:14,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:14,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:15,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:16,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:17,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:18,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:19,874 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:20,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:20,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:21,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:22,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:23,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:24,875 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:25,753 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:25,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:26,878 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:27,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:28,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:29,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:34:29,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:34:29,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:30,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:30,882 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:31,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:34:31,255 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:34:31,256 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:34:31,257 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:34:31,792 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:34:31,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:32,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:33,793 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:34:33,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:34,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:35,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:36,704 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:34:36,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:36,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:37,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:38,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:39,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:40,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:41,751 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:41,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:42,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:43,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:44,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:34:44,167 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:34:44,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:45,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:46,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:46,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:47,880 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:48,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:49,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:50,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:51,752 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:51,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:52,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:53,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:54,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:55,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:56,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:34:56,894 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:57,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:58,888 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:34:59,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:34:59,171 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:34:59,887 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:00,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:01,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:01,891 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:02,496 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:35:02,498 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:35:02,498 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:35:02,500 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:35:02,822 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:35:02,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:03,823 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:35:03,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:04,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:05,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:06,707 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:35:06,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:07,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:07,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:08,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:09,899 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:10,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:11,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:12,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:12,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:13,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:14,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:35:14,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:35:14,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:15,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:16,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:17,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:18,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:18,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:19,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:20,900 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:21,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:22,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:23,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:23,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:24,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:25,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:26,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:27,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:28,901 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:29,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:35:29,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:35:29,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:29,918 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:30,906 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:31,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:32,909 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:33,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:34,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:34,919 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:35,881 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:35:35,882 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:35:35,883 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:35:35,884 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:35:36,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:36,711 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:35:36,850 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:35:37,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:37,851 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:35:38,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:39,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:40,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:40,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:41,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:42,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:43,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:44,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:44,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:35:44,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:35:45,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:46,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:46,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:47,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:48,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:49,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:50,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:51,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:51,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:52,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:53,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:54,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:55,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:56,076 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:56,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:35:57,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:58,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:59,077 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:35:59,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:35:59,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:36:00,082 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:01,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:01,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:02,078 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:03,083 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:04,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:05,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:05,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:36:05,626 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:36:05,627 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:36:05,628 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:36:05,874 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:36:06,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:06,714 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:36:07,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:07,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:07,876 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:36:08,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:09,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:10,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:11,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:12,084 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:12,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:13,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:14,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:14,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:36:14,168 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:36:15,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:16,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:17,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:18,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:18,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:19,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:20,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:21,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:22,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:23,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:23,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:24,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:25,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:26,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:27,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:28,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:28,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:29,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:29,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:36:29,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:36:30,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:31,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:32,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:33,089 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:33,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:34,090 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:35,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:36,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:36,726 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:36:37,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:38,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:38,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:36:38,592 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:36:38,592 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:36:38,594 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:36:38,905 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:36:39,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:39,632 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:39,906 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:36:40,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:41,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:42,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:43,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:44,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:44,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:36:44,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:36:44,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:45,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:46,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:47,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:48,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:49,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:49,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:50,094 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:51,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:52,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:53,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:54,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:54,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:36:55,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:56,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:57,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:58,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:36:59,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:36:59,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:36:59,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:00,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:00,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:01,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:02,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:03,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:04,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:05,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:05,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:06,224 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:06,724 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:37:07,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:08,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:09,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:09,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:37:09,249 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:37:09,249 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:37:09,254 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:37:09,932 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:37:09,932 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:37:10,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:11,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:11,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:12,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:13,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:14,168 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:37:14,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:37:14,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:15,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:16,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:17,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:17,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:18,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:19,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:20,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:21,227 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:22,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:22,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:23,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:24,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:25,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:26,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:27,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:27,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:28,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:29,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:37:29,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:37:29,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:30,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:31,229 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:32,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:33,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:33,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:34,241 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:35,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:36,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:36,735 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:37:37,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:38,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:39,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:39,736 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:40,136 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:37:40,138 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:37:40,138 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:37:40,144 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:37:40,965 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:37:41,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:41,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:41,966 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:37:42,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:43,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:44,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:37:44,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:37:44,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:44,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:45,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:46,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:47,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:48,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:49,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:50,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:50,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:51,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:52,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:53,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:54,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:55,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:55,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:37:56,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:57,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:58,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:37:59,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:37:59,169 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:37:59,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:00,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:01,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:01,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:02,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:03,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:04,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:05,454 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:06,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:06,741 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:38:07,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:07,747 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:08,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:09,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:10,012 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:38:10,013 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:38:10,014 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:38:10,015 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:38:10,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:10,993 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:38:11,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:11,994 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:38:12,809 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:13,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:13,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:14,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:38:14,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:38:14,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:15,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:16,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:17,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:18,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:18,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:19,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:20,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:21,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:22,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:23,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:24,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:24,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:25,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:26,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:27,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:28,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:29,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:38:29,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:38:29,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:30,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:30,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:31,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:32,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:33,814 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:34,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:35,782 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:35,813 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:36,738 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:38:36,820 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:37,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:38,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:38,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:38:38,844 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:38:38,844 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:38:38,846 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:38:39,018 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:38:39,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:40,019 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:38:40,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:41,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:41,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:42,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:43,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:44,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:38:44,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:38:44,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:45,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:46,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:47,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:47,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:48,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:49,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:51,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:52,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:52,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:53,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:54,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:55,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:56,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:57,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:57,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:38:58,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:59,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:38:59,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:38:59,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:39:00,056 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:01,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:02,057 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:02,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:03,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:04,065 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:05,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:06,064 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:06,750 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:39:07,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:08,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:08,510 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:39:08,511 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:39:08,511 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:39:08,512 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:08,513 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:39:09,048 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:39:09,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:10,049 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:39:10,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:11,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:12,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:13,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:13,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:14,062 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:14,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:39:14,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:39:15,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:16,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:17,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:18,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:18,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:19,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:20,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:21,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:22,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:23,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:24,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:24,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:25,143 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:26,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:27,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:28,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:29,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:29,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:39:29,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:39:30,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:30,788 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:31,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:32,144 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:33,154 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:34,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:35,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:35,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:36,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:36,749 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:39:37,149 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:38,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:38,679 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:39:38,681 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:39:38,681 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:39:38,687 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:39:39,075 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:39:39,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:40,076 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:39:40,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:41,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:41,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:42,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:43,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:44,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:44,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:39:44,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:39:45,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:46,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:47,150 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:47,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:48,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:49,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:50,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:51,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:52,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:52,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:53,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:54,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:55,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:56,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:57,151 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:58,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:58,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:39:59,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:39:59,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:39:59,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:40:00,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:01,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:02,152 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:03,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:03,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:04,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:05,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:06,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:06,751 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:40:07,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:08,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:09,129 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:40:09,130 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:40:09,131 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:40:09,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:09,136 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:40:09,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:10,105 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:40:10,106 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:40:10,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:11,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:12,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:13,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:14,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:40:14,170 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:40:14,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:14,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:15,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:16,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:17,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:18,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:19,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:19,794 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:20,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:21,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:22,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:23,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:24,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:24,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:25,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:26,280 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:27,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:28,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:29,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:40:29,171 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:40:29,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:30,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:30,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:31,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:32,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:33,281 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:34,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:35,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:35,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:36,293 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:36,761 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:40:37,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:38,291 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:39,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:39,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:40:39,556 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:40:39,556 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:40:39,562 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:40:40,143 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:40:40,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:40,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:41,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:42,145 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:40:42,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:43,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:44,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:40:44,171 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:40:44,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:45,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:46,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:46,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:47,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:48,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:49,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:50,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:51,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:52,290 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:52,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:53,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:54,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:55,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:56,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:57,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:57,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:40:58,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:40:59,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:40:59,171 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:40:59,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:00,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:01,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:02,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:03,373 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:03,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:04,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:05,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:06,374 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:06,767 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:41:07,389 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:08,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:09,380 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:09,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:10,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:11,383 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:12,387 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:13,119 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:41:13,120 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:41:13,121 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:41:13,126 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:41:13,181 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:41:13,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:14,180 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:41:14,181 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:41:14,188 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:41:14,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:14,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:15,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:16,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:17,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:18,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:19,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:20,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:20,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:21,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:22,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:23,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:24,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:25,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:26,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:26,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:27,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:28,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:29,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:41:29,171 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:41:29,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:30,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:31,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:31,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:32,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:33,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:34,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:35,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:36,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:36,770 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:41:37,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:37,772 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:38,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:39,557 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:40,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:41,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:42,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:42,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:43,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:44,109 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:41:44,110 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:41:44,111 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:41:44,114 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:41:44,210 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:41:44,536 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:41:44,537 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:41:44,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:45,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:46,212 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:41:46,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:47,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:48,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:48,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:49,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:50,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:51,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:52,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:53,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:54,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:54,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:41:55,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:56,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:57,656 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:58,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:41:59,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:41:59,260 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:41:59,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:00,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:00,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:01,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:02,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:03,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:04,658 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:05,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:05,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:06,657 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:06,774 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:42:07,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:08,661 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:09,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:10,664 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:11,666 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:11,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:12,665 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:13,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:42:13,339 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:42:13,340 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:42:13,341 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:42:13,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:14,238 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:42:14,238 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:42:14,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:42:14,260 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:42:14,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:15,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:16,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:17,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:17,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:18,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:19,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:20,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:21,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:22,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:23,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:23,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:24,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:25,755 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:26,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:27,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:28,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:28,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:29,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:42:29,260 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:42:29,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:30,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:31,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:32,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:33,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:34,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:34,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:35,756 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:36,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:36,778 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:42:37,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:38,757 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:39,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:39,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:40,758 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:41,759 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:42,760 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:43,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:44,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:42:44,268 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:42:44,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:45,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:45,922 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:42:45,924 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:42:45,924 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:42:45,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:45,929 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:42:46,270 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:42:46,761 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:47,271 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:42:47,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:48,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:49,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:50,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:50,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:51,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:52,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:53,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:54,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:55,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:56,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:56,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:42:57,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:58,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:42:59,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:42:59,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:42:59,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:00,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:01,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:02,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:02,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:03,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:04,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:05,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:06,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:06,783 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:43:07,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:08,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:08,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:09,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:10,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:11,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:12,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:13,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:13,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:14,267 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:43:14,267 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:43:14,765 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:15,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:16,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:17,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:18,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:19,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:19,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:20,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:43:20,035 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:43:20,035 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:43:20,037 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:43:20,301 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:43:20,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:21,302 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:43:21,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:22,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:23,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:24,768 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:24,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:25,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:26,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:27,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:28,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:29,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:43:29,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:43:29,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:30,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:30,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:31,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:32,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:33,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:34,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:35,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:35,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:36,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:36,787 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:43:37,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:38,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:39,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:40,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:41,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:41,981 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:42,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:43,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:44,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:43:44,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:43:44,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:45,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:46,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:46,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:47,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:48,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:49,774 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:50,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:51,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:51,995 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:52,122 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:43:52,123 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:43:52,124 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:43:52,127 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:43:52,333 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:43:52,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:53,334 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:43:53,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:54,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:55,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:56,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:57,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:57,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:43:58,775 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:43:59,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:43:59,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:43:59,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:00,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:01,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:02,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:02,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:03,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:04,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:05,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:06,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:06,791 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:44:07,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:08,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:08,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:09,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:10,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:11,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:12,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:13,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:13,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:14,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:44:14,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:44:14,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:15,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:16,784 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:17,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:18,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:18,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:19,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:20,785 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:21,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:22,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:44:22,023 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:44:22,023 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:44:22,025 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:44:22,362 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:44:22,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:23,363 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:44:23,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:24,063 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:24,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:25,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:26,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:27,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:28,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:29,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:44:29,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:44:29,343 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:29,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:30,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:31,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:32,786 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:33,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:34,959 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:34,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:35,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:36,795 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:44:36,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:37,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:38,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:39,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:40,863 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:40,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:41,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:42,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:43,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:44,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:44:44,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:44:44,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:45,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:45,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:46,864 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:47,869 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:48,873 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:49,870 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:50,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:50,994 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:51,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:52,871 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:53,648 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:44:53,649 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:44:53,649 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:44:53,651 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:44:54,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:54,386 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:44:55,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:55,387 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:44:56,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:56,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:44:57,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:58,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:59,060 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:44:59,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:44:59,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:45:00,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:01,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:01,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:02,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:03,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:04,061 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:05,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:06,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:06,800 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:45:07,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:07,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:08,159 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:09,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:10,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:11,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:12,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:12,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:13,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:14,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:14,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:45:14,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:45:15,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:16,160 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:17,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:17,996 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:18,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:19,164 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:20,169 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:21,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:22,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:22,674 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:45:22,676 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:45:22,676 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:45:22,677 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:45:23,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:23,417 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:45:23,716 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:24,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:25,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:25,419 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:45:26,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:27,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:28,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:28,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:29,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:29,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:45:29,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:45:30,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:31,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:32,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:33,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:34,313 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:34,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:35,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:36,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:36,805 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:45:37,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:38,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:39,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:40,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:40,998 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:41,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:42,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:43,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:44,218 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:44,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:45:44,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:45:45,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:46,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:46,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:47,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:48,223 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:49,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:50,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:51,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:51,101 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:45:51,103 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:45:51,103 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:45:51,105 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:45:51,445 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:45:51,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:52,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:53,447 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:45:53,518 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:54,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:55,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:56,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:57,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:45:57,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:58,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:45:59,265 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:45:59,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:45:59,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:00,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:01,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:02,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:03,001 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:03,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:04,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:05,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:06,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:06,808 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:46:07,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:08,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:08,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:09,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:10,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:11,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:12,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:13,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:14,002 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:14,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:46:14,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:46:14,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:15,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:16,524 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:17,527 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:18,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:19,009 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:19,525 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:20,526 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:21,161 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:46:21,162 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:46:21,163 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:46:21,164 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:46:21,474 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:46:21,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:22,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:23,475 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:46:23,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:24,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:24,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:25,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:26,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:27,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:28,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:29,266 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:46:29,266 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:46:29,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:29,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:30,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:31,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:32,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:33,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:34,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:35,003 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:35,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:36,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:36,812 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:46:37,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:38,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:39,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:40,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:41,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:41,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:42,577 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:43,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:44,279 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:46:44,280 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:46:44,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:45,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:46,007 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:46,590 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:47,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:48,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:48,845 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:46:48,845 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:46:48,846 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:46:48,848 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:46:49,501 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:46:49,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:50,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:51,503 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:46:51,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:52,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:52,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:53,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:54,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:55,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:56,587 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:57,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:58,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:46:58,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:46:59,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:46:59,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:46:59,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:00,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:01,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:02,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:03,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:04,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:04,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:05,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:06,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:06,816 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:47:07,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:08,588 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:09,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:09,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:10,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:11,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:12,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:13,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:14,271 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:47:14,275 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:47:14,595 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:15,014 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:15,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:16,597 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:17,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:18,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:19,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:19,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:47:19,603 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:47:19,604 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:47:19,605 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:47:20,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:20,527 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:47:20,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:21,528 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:47:21,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:22,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:23,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:24,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:25,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:26,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:26,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:27,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:28,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:29,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:47:29,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:47:29,598 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:30,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:31,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:31,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:32,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:33,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:34,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:35,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:36,022 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:36,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:36,819 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:47:37,599 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:38,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:39,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:40,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:41,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:42,021 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:42,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:43,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:44,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:47:44,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:47:44,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:45,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:46,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:47,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:47,603 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:48,607 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:49,611 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:50,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:50,938 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:47:50,939 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:47:50,939 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:47:50,941 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:47:51,557 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:47:51,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:52,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:52,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:53,558 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:47:53,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:54,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:55,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:56,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:57,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:58,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:47:58,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:47:59,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:47:59,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:47:59,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:00,608 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:01,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:02,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:03,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:03,609 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:04,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:05,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:06,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:06,823 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:48:07,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:08,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:08,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:09,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:10,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:11,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:12,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:13,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:14,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:14,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:48:14,272 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:48:14,685 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:15,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:16,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:17,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:18,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:19,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:19,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:20,686 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:21,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:22,690 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:22,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:48:22,963 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:48:22,964 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:48:22,965 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:48:23,583 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:48:23,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:24,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:25,004 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:25,584 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:48:25,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:26,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:27,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:28,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:29,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:48:29,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:48:29,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:30,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:30,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:31,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:32,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:33,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:34,691 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:35,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:36,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:36,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:36,827 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:48:37,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:38,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:39,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:40,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:41,029 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:41,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:42,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:43,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:44,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:48:44,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:48:44,692 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:45,693 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:46,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:47,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:47,699 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:48,698 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:49,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:50,701 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:51,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:52,025 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:48:52,026 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:48:52,026 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:48:52,028 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:48:52,066 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:52,615 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:48:52,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:53,616 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:48:53,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:54,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:55,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:56,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:57,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:58,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:48:58,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:48:59,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:48:59,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:48:59,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:00,702 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:01,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:02,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:03,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:04,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:04,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:05,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:06,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:06,831 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:49:07,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:08,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:09,703 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:09,834 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:10,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:11,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:12,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:13,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:14,272 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:49:14,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:49:14,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:15,036 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:15,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:16,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:17,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:18,704 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:19,714 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:20,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:20,706 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:21,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:22,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:23,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:24,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:24,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:49:24,809 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:49:24,810 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:49:24,811 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:49:25,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:25,658 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:49:25,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:26,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:27,659 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:49:27,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:28,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:29,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:49:29,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:49:29,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:30,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:31,216 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:31,707 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:32,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:33,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:34,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:35,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:36,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:36,835 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:49:36,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:37,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:38,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:39,708 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:40,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:41,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:42,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:42,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:43,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:44,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:49:44,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:49:44,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:45,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:46,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:47,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:48,217 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:48,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:49,709 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:50,710 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:51,715 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:52,711 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:53,228 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:53,712 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:54,717 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:55,718 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:56,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:56,738 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:49:56,739 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:49:56,739 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:49:56,741 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:49:57,689 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:49:57,690 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:49:57,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:58,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:49:58,779 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:49:59,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:49:59,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:49:59,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:00,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:01,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:02,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:03,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:04,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:04,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:05,719 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:06,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:06,839 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:50:07,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:08,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:09,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:09,842 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:10,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:11,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:12,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:13,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:14,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:50:14,274 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:50:14,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:15,221 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:15,720 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:16,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:17,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:18,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:19,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:20,222 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:20,721 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:21,727 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:22,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:23,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:24,724 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:25,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:25,725 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:26,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:27,463 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:50:27,464 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:50:27,465 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:50:27,466 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:50:27,718 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:50:27,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:28,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:29,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:50:29,274 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:50:29,719 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:50:29,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:30,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:30,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:31,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:32,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:33,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:34,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:35,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:36,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:36,844 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:50:36,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:37,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:38,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:39,890 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:41,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:42,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:42,225 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:43,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:44,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:44,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:50:44,273 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:50:45,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:46,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:47,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:47,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:48,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:49,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:50,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:51,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:52,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:53,183 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:53,226 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:54,195 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:55,187 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:56,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:57,190 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:58,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:58,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:50:59,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:50:59,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:50:59,275 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:50:59,529 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:50:59,531 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:50:59,532 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:50:59,533 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:50:59,748 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:51:00,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:01,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:01,750 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:51:02,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:03,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:04,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:04,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:05,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:06,191 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:06,849 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:51:07,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:08,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:09,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:09,851 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:10,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:11,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:12,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:13,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:14,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:14,273 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:51:14,274 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:51:15,192 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:15,230 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:16,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:17,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:18,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:19,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:20,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:20,231 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:21,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:22,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:23,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:24,193 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:25,201 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:25,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:26,196 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:27,204 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:28,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:29,206 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:29,277 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:51:29,277 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:51:30,205 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:31,145 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:51:31,146 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:51:31,147 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:51:31,147 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:31,148 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:51:31,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:31,785 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:51:32,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:33,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:33,787 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:51:34,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:35,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:36,234 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:36,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:36,852 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:51:37,573 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:38,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:39,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:40,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:41,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:42,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:42,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:43,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:44,274 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:51:44,275 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:51:44,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:45,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:46,574 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:47,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:47,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:48,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:49,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:50,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:51,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:52,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:53,235 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:53,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:54,575 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:55,586 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:56,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:57,582 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:58,237 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:51:58,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:51:59,275 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:51:59,276 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:51:59,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:00,583 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:01,342 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:52:01,343 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:52:01,344 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:52:01,346 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:52:01,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:01,819 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:52:02,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:03,384 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:03,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:03,821 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:52:04,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:05,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:06,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:06,856 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:52:07,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:08,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:08,858 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:09,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:10,762 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:11,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:12,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:13,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:14,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:14,276 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:52:14,276 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:52:14,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:15,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:16,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:17,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:18,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:19,240 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:19,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:20,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:21,763 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:22,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:23,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:24,764 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:25,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:25,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:26,766 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:27,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:28,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:29,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:52:29,279 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:52:29,769 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:30,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:31,091 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:52:31,093 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:52:31,093 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:52:31,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:31,095 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:52:31,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:31,846 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:52:32,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:33,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:33,847 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:52:34,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:35,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:36,243 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:36,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:36,860 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:52:37,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:38,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:39,770 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:40,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:41,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:42,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:42,771 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:43,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:44,278 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:52:44,279 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:52:44,810 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:45,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:46,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:47,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:47,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:48,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:49,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:50,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:51,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:52,244 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:52,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:53,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:54,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:55,812 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:56,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:57,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:52:57,815 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:58,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:52:59,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:52:59,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:52:59,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:00,816 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:01,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:02,121 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:53:02,122 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:53:02,123 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:53:02,124 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:53:02,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:02,876 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:53:03,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:03,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:03,877 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:53:04,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:05,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:06,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:06,864 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:53:07,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:08,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:08,867 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:09,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:10,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:11,817 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:12,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:13,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:14,247 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:14,282 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:53:14,283 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:53:14,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:15,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:16,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:17,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:18,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:19,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:19,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:20,818 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:21,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:22,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:23,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:24,248 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:24,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:25,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:26,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:27,819 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:28,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:29,253 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:29,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:53:29,287 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:53:29,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:30,823 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:31,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:32,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:33,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:34,203 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:53:34,204 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:53:34,205 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:53:34,206 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:53:34,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:34,907 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:53:35,245 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:35,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:35,908 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:53:36,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:36,869 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:53:37,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:38,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:39,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:40,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:40,825 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:41,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:42,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:43,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:44,283 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:53:44,284 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:53:44,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:45,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:45,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:46,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:47,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:48,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:49,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:50,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:50,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:51,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:52,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:53,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:54,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:55,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:56,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:53:56,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:57,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:58,831 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:53:59,284 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:53:59,285 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:53:59,839 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:00,832 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:01,257 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:01,835 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:02,833 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:03,838 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:04,489 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:54:04,491 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:54:04,492 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:54:04,493 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:54:04,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:04,945 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:54:05,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:05,946 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:54:06,872 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:54:06,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:06,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:07,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:08,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:09,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:10,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:11,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:12,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:12,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:13,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:14,285 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:54:14,285 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:54:14,907 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:15,908 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:16,930 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:17,254 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:17,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:18,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:19,927 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:20,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:21,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:22,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:22,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:23,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:24,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:25,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:26,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:27,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:28,255 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:28,928 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:29,296 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:54:29,301 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:54:29,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:30,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:31,932 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:32,937 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:33,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:33,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:34,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:35,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:54:35,021 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:54:35,021 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:54:35,023 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:54:35,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:35,971 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:54:35,972 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:54:36,876 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:54:36,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:37,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:38,879 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:38,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:39,934 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:40,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:41,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:42,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:43,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:44,287 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:54:44,288 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:54:44,413 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:44,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:45,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:46,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:47,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:48,935 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:49,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:49,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:50,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:51,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:52,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:53,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:54,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:54,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:55,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:56,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:57,968 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:58,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:54:59,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:54:59,288 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:54:59,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:54:59,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:00,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:01,969 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:02,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:03,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:04,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:05,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:05,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:06,881 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:55:06,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:07,619 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:55:07,620 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:55:07,620 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:55:07,622 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:55:08,000 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:55:08,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:09,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:10,002 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:55:10,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:10,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:11,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:12,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:13,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:14,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:14,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:55:14,288 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:55:15,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:15,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:16,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:17,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:18,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:19,033 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:20,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:20,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:21,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:22,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:23,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:24,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:25,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:26,034 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:26,436 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:27,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:28,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:29,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:29,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:55:29,288 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:55:30,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:31,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:32,040 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:32,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:33,038 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:34,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:35,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:36,042 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:36,886 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:55:37,043 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:37,671 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:55:37,672 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:55:37,673 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:55:37,673 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:37,674 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:55:38,028 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:55:38,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:39,085 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:40,029 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:55:40,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:41,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:42,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:43,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:43,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:44,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:44,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:55:44,288 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:55:45,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:46,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:47,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:48,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:49,086 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:49,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:50,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:51,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:52,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:53,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:54,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:54,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:55:55,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:56,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:57,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:58,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:59,087 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:55:59,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:55:59,288 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:55:59,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:00,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:01,088 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:02,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:03,092 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:04,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:05,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:05,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:06,095 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:06,895 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:56:07,097 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:08,015 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:56:08,016 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:56:08,016 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:56:08,017 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:56:08,066 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:56:08,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:09,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:10,068 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:56:10,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:11,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:11,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:12,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:13,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:14,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:14,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:56:14,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:56:15,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:16,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:17,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:17,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:18,177 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:19,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:20,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:21,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:22,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:22,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:23,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:24,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:25,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:26,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:27,178 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:28,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:28,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:29,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:29,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:56:29,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:56:30,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:31,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:32,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:33,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:34,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:34,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:35,179 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:36,184 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:36,894 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:56:37,181 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:38,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:39,185 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:39,896 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:40,189 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:41,182 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:41,186 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:56:41,188 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:56:41,188 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:56:41,194 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:56:42,105 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:56:42,105 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:56:42,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:43,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:44,288 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:56:44,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:56:44,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:45,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:45,422 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:46,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:47,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:48,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:49,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:50,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:50,446 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:51,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:52,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:53,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:54,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:55,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:55,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:56:56,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:57,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:58,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:56:59,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:56:59,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:56:59,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:00,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:01,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:01,447 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:02,398 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:03,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:04,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:05,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:06,399 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:06,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:06,909 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:57:07,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:08,401 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:09,404 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:10,402 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:11,356 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:57:11,356 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:57:11,357 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:57:11,361 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:57:12,131 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:57:12,131 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:57:12,397 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:12,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:13,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:14,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:57:14,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:57:14,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:15,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:16,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:17,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:17,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:18,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:19,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:20,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:21,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:22,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:22,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:23,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:24,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:25,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:26,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:27,460 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:28,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:28,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:29,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:57:29,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:57:29,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:30,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:31,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:32,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:33,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:34,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:34,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:35,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:36,461 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:36,897 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:57:37,471 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:38,464 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:39,465 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:39,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:40,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:41,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:42,467 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:43,327 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:57:43,328 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:57:43,329 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:57:43,332 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:57:43,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:44,160 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:57:44,160 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:57:44,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:57:44,289 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:57:44,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:45,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:45,744 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:46,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:47,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:48,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:49,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:50,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:50,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:51,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:52,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:53,745 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:54,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:55,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:56,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:57:56,746 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:57,778 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:58,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:57:59,289 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:57:59,290 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:57:59,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:00,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:01,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:02,453 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:02,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:03,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:04,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:05,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:06,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:06,901 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:58:07,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:07,904 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:08,777 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:09,787 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:10,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:11,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:12,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:13,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:13,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:14,298 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:58:14,298 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:58:14,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:15,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:58:15,549 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:58:15,549 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:58:15,555 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:58:15,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:16,189 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:58:16,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:17,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:18,190 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:58:18,589 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:18,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:19,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:20,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:21,971 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:22,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:23,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:24,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:24,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:25,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:26,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:27,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:28,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:29,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:58:29,295 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:58:29,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:30,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:30,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:31,972 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:32,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:33,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:34,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:35,459 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:35,973 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:36,909 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:58:36,980 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:37,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:38,974 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:39,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:40,466 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:40,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:41,978 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:42,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:42,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 21:58:42,990 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 21:58:42,990 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 21:58:42,992 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 21:58:43,212 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 21:58:43,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:44,213 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:58:44,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:58:44,295 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:58:44,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:45,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:46,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:46,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:47,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:48,975 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:49,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:50,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:51,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:51,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:52,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:53,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:54,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:55,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:56,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:58:56,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:57,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:58,976 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:58:59,295 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:58:59,295 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:58:59,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:00,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:01,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:02,462 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:02,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:03,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:04,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:06,010 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:06,910 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:59:07,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:08,112 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:08,228 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 21:59:12,914 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:14,841 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:15,963 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:59:15,964 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:59:16,389 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:18,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:22,783 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:23,479 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:24,807 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:28,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:31,533 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:32,913 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:32,965 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:59:32,966 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:59:34,035 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:36,914 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 21:59:39,584 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:39,916 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:40,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:45,219 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:47,547 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:49,338 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 21:59:49,339 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 21:59:49,847 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 21:59:50,485 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 21:59:55,486 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:00,487 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:00,600 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:00,641 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:01,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:02,600 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:00:02,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:02,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:00:02,677 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:00:03,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:04,601 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:05,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:05,859 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:06,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:06,918 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:00:07,602 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:08,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:09,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:10,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:10,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:11,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:12,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:13,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:14,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:15,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:16,019 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:16,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:17,622 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:17,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:00:17,677 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:00:18,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:19,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:20,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:21,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:22,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:22,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:23,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:24,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:25,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:26,623 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:27,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:27,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:28,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:29,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:30,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:31,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:32,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:32,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:00:32,678 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:00:32,776 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:33,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:34,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:35,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:36,624 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:36,922 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:00:37,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:37,924 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:38,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:39,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:40,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:41,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:42,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:43,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:43,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:44,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:45,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:46,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:47,625 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:47,677 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:00:47,677 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:00:48,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:48,748 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:49,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:50,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:51,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:52,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:53,626 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:00:54,020 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:00:59,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:04,026 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:06,926 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:01:09,929 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:13,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:01:13,987 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:01:14,027 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:14,982 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:15,093 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:15,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:16,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:17,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:18,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:19,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:20,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:20,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:21,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:22,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:23,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:24,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:25,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:26,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:26,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:27,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:28,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:01:28,983 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:01:29,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:29,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:30,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:31,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:32,125 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:32,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:33,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:34,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:35,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:36,930 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:01:36,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:37,933 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:37,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:38,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:39,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:40,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:41,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:42,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:42,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:43,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:01:43,983 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:01:44,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:44,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:45,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:46,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:47,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:48,120 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:48,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:49,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:50,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:51,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:52,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:53,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:53,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:54,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:55,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:56,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:57,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:58,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:01:58,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:01:58,984 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:01:59,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:01:59,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:00,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:01,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:02,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:03,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:04,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:04,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:05,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:06,953 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:02:06,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:07,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:08,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:09,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:09,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:10,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:11,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:12,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:13,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:02:13,983 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:02:14,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:14,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:15,080 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:15,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:16,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:17,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:18,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:19,988 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:20,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:20,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:21,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:22,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:23,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:24,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:25,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:26,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:26,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:27,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:28,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:02:28,983 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:02:29,023 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:29,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:30,989 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:31,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:32,146 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:32,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:33,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:34,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:35,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:36,939 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:02:36,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:37,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:37,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:38,990 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:39,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:40,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:41,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:42,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:42,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:43,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:02:43,983 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:02:44,024 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:44,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:45,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:46,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:47,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:48,148 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:48,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:49,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:50,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:51,991 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:52,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:53,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:02:53,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:54,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:55,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:02:58,985 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:03,986 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:06,947 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:03:09,950 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:14,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:19,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:24,955 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:29,956 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:34,957 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:36,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:03:36,412 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:36,412 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:03:36,947 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:03:37,406 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:38,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:39,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:40,364 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:40,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:40,995 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:03:41,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:42,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:43,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:44,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:44,998 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:03:45,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:45,992 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:46,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:47,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:48,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:49,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:50,791 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:51,409 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:03:51,410 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:03:51,542 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:51,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:52,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:53,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:54,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:55,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:56,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:57,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:03:57,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:58,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:03:59,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:00,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:01,792 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:02,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:03,411 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:03,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:04,793 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:05,802 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:06,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:04:06,418 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:04:06,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:06,953 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:04:07,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:08,797 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:08,962 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:09,799 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:10,983 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:11,264 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:04:11,265 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:04:11,265 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:04:11,267 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:04:12,023 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:04:12,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:13,024 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:04:13,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:14,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:14,252 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:15,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:16,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:17,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:18,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:19,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:19,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:20,249 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:21,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:21,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:04:21,415 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:04:22,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:23,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:24,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:24,483 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:25,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:26,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:27,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:28,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:29,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:30,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:30,421 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:31,250 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:32,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:33,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:34,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:35,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:36,251 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:36,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:04:36,415 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:04:36,550 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:36,965 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:04:37,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:38,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:39,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:40,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:41,256 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:42,260 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:42,435 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:43,013 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:04:43,014 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:04:43,014 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:04:43,016 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:04:43,047 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:04:43,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:44,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:45,049 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:04:45,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:46,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:47,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:48,426 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:48,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:49,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:50,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:51,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:04:51,415 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:04:51,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:52,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:53,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:53,538 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:54,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:55,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:56,430 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:57,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:58,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:04:59,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:04:59,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:00,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:01,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:02,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:03,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:04,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:05,427 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:05,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:06,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:05:06,415 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:05:06,455 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:06,959 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:05:07,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:08,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:09,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:10,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:10,964 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:11,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:12,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:13,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:14,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:15,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:15,795 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:05:15,796 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:05:15,796 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:05:15,802 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:05:16,083 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:05:16,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:16,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:17,085 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:05:17,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:18,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:19,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:20,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:21,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:05:21,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:05:21,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:21,567 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:22,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:23,442 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:24,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:25,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:26,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:27,428 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:27,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:28,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:29,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:30,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:31,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:32,429 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:32,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:33,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:34,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:35,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:36,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:05:36,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:05:36,456 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:36,964 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:05:37,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:37,966 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:38,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:39,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:40,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:41,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:42,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:43,432 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:43,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:44,448 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:45,393 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:05:45,394 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:05:45,394 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:05:45,396 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:05:45,811 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:46,108 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:05:46,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:47,109 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:05:47,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:48,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:49,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:49,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:50,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:51,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:05:51,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:05:51,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:52,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:53,543 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:54,519 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:05:54,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:55,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:56,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:57,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:58,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:05:59,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:00,431 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:00,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:01,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:02,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:03,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:04,544 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:05,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:06,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:06:06,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:06:06,495 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:06,545 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:06,968 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:06:07,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:08,551 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:09,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:10,576 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:11,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:12,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:12,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:13,516 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:06:13,517 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:06:13,518 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:06:13,520 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:06:13,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:14,134 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:06:14,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:15,135 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:06:15,939 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:16,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:17,434 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:17,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:18,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:19,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:20,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:21,415 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:06:21,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:06:21,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:22,493 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:22,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:23,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:24,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:25,940 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:26,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:27,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:28,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:28,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:29,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:30,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:31,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:32,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:33,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:34,433 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:34,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:35,941 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:36,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:06:36,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:06:36,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:36,972 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:06:37,942 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:38,951 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:39,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:39,977 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:40,945 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:41,946 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:42,948 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:43,947 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:44,713 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:06:44,714 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:06:44,715 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:06:44,716 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:06:45,131 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:45,168 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:06:45,754 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:46,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:47,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:47,169 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:06:48,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:49,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:50,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:51,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:51,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:06:51,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:06:51,520 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:52,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:53,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:54,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:55,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:56,130 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:57,165 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:57,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:06:58,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:06:59,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:00,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:01,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:02,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:03,162 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:03,440 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:04,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:05,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:06,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:06,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:07:06,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:07:06,977 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:07:07,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:08,163 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:08,984 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:09,173 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:10,166 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:11,167 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:12,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:13,170 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:14,171 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:14,443 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:14,889 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:07:14,890 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:07:14,891 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:07:14,892 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:07:15,199 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:07:15,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:16,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:17,202 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:07:17,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:18,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:19,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:20,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:20,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:21,310 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:21,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:07:21,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:07:22,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:23,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:24,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:25,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:25,552 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:26,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:27,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:28,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:29,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:30,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:31,311 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:31,441 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:32,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:33,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:34,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:35,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:36,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:36,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:07:36,416 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:07:36,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:36,981 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:07:37,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:38,312 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:39,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:40,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:41,316 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:42,321 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:42,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:43,322 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:44,324 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:45,138 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:07:45,138 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:07:45,139 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:07:45,140 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:07:45,232 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:07:45,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:46,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:47,234 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:07:47,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:47,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:48,553 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:49,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:50,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:51,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:07:51,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:07:51,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:52,514 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:52,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:53,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:54,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:55,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:56,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:57,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:58,444 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:07:58,554 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:07:59,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:00,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:01,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:02,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:03,445 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:03,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:04,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:05,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:06,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:08:06,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:08:06,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:06,985 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:08:07,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:08,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:08,987 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:09,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:10,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:11,556 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:12,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:13,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:14,450 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:14,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:15,558 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:16,559 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:17,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:18,117 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:08:18,118 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:08:18,118 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:08:18,120 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:08:18,264 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:08:18,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:19,265 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:08:19,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:20,158 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:20,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:21,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:08:21,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:08:21,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:22,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:23,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:24,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:25,555 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:25,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:26,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:27,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:28,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:29,561 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:30,726 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:31,449 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:31,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:32,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:33,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:34,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:35,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:36,417 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:08:36,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:08:36,560 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:36,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:36,989 DEBUG SenderThread:823089 [sender.py:send():379] send: stats +2024-12-26 22:08:37,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:38,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:39,723 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:40,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:41,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:42,451 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:42,730 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:43,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:44,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:45,734 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:46,630 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 22:08:46,631 DEBUG SenderThread:823089 [sender.py:send():379] send: history +2024-12-26 22:08:46,632 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: summary_record +2024-12-26 22:08:46,633 INFO SenderThread:823089 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 22:08:47,048 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:47,292 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/wandb-summary.json +2024-12-26 22:08:47,672 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:47,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:48,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:49,294 INFO Thread-12 :823089 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241226_145403-c7ovtvny/files/output.log +2024-12-26 22:08:49,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:50,780 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:51,416 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 22:08:51,417 DEBUG SenderThread:823089 [sender.py:send_request():406] send_request: stop_status +2024-12-26 22:08:51,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:52,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:53,568 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:53,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:54,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:55,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:56,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:57,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:58,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:08:59,452 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 22:08:59,781 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:09:00,921 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:09:01,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:09:02,826 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-26 22:09:03,827 DEBUG HandlerThread:823089 [handler.py:handle_request():158] handle_request: internal_messages