PLDRv51_110M_3_hyperparameters.py

import torch.nn.functional as F

hpdict={'num_layers': 5,
        'd_model': 896,
        'num_heads': 14,
        'dff': 2389,
        'A_dff': 180,
        'num_reslayerA': 8,
        'num_denseA': 2,
        'input_vocab_size': 32000,
        'max_seq_len': 1024,
        'epochs': 1,
        'save_model_path': './PLDRv51-110M-3-checkpoint',
        'warmup_steps': 2000,
        'lr_total_steps': 250000,
        'learning_rate': 0.001,
        'lr_alpha': 0.1,
        'adamw_decay': 0.1,
        'activation': F.silu,
        'disable_amp': False,
        'auto_size_minimum': None,
        'disable_fsdp_mixed_precision': False,
        'fsdp_cpu_offload': False,
        'fsdp_sharding_strategy': 'HYBRID_SHARD',
        'backward_prefetch': 'PRE',
        'save_type': 'torch'}