##Offloading Activations/Weights to CPU
  cpu_offloading: False
  cpu_offloading_num_layers: ${sum:${.num_layers},-1} #This value should be between [1,num_layers-1] as we don't want to offload the final layer's activations and expose any offloading duration for the final layer
  cpu_offloading_activations: True
  cpu_offloading_weights: True

  reset_lr: False # Set to True to reset learning rate to initial learning rate. Only supported with distributed optmizer and megatron_amp_O2.
  reset_lr_steps: False # Set to True to adjust learning rate's max_steps and decay_steps by subtracting number of steps already completed at the checkpoint.

mcore_gpt: True

rampup_batch_size: null # Should be a list of 3 values:
encoder_seq_length: 1024

megatron_amp_O2: True # Enable O2-level automatic mixed precision using main parameters
grad_allreduce_chunk_size_mb: 125

grad_div_ar_fusion: True # Fuse grad division into torch.distributed.all_reduce. Only used with O2 and no pipeline parallelism..
get_attention_mask_from_fusion: True # When using fused softmax it will create the attention mask so we won't copy it to the pipeline stages.

seed: 1234
apex_transformer_log_level: 30 # Python logging level displays logs with severity greater than or equal to this
gradient_as_bucket_view: True # PyTorch DDP argument. Allocate gradients in a contiguous bucket to save memory (less fragmentation and buffer memory)

data_impl: mmap
mmap_bin_files: True
skip_warmup: True
dataloader_type: single # cyclic
validation_drop_last: True # Set to false if the last partial validation samples is to be consumed
shuffle_documents: True # Set to False to disable documents shuffling. Sample index will still be shuffled