pretrain core 0

scripts/pretrain-core-model-0.yaml

@@ -64,7 +64,7 @@ train:
   # global_batch_size: 64
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  micro_batch_size: 6
+  micro_batch_size: 5
   # micro_batch_size: 4
   # micro_batch_size: 2
   # micro_batch_size: 1
@@ -113,8 +113,8 @@ eval:
 # Optimizer-related arguments
 
 optimizer:
-  # class_path: torch.optim.AdamW
-  class_path: torchao.prototype.low_bit_optim.AdamW8bit
+  class_path: torch.optim.AdamW
+  # class_path: torchao.prototype.low_bit_optim.AdamW8bit
   # class_path: torchao.prototype.low_bit_optim.AdamW4bit
   # class_path: bitsandbytes.optim.AdamW8bit
   # class_path: bitsandbytes.optim.PagedAdamW8bit