llama2_70b.toml

# torchtitan Config.toml
# NOTE: this toml config is a preset for 64 A100 GPUs.

[job]
dump_folder = "./outputs"
description = "Llama2 70B training"

[profiling]
enable_profiling = true
save_traces_folder = "profile_trace"
profile_freq = 100

[metrics]
log_freq = 1
enable_tensorboard = false
save_tb_folder = "tb"

[model]
name = "llama2"
flavor = "70B"
norm_type = "fused_rmsnorm"  # layernorm / np_layernorm / rmsnorm / fused_rmsnorm
tokenizer_path = "/mnt/petrelfs/share_data/yanhang/tokenizes/v13.model"

[optimizer]
name = "AdamW"
lr = 1e-4

[training]
batch_size = 2
seq_len = 4096
warmup_steps = 10  # lr scheduler warm up, 0.01 of the train steps
max_norm = 1.0  # grad norm clipping
steps = 1000
data_parallel_replicate_degree = 1
data_parallel_shard_degree = -1
tensor_parallel_degree = 1
compile = false
dataset = "tinystories"

[experimental]
context_parallel_degree = 1
pipeline_parallel_degree = 1

[checkpoint]
enable_checkpoint = false
folder = "checkpoint"
interval_type = "steps"
interval = 500
model_weights_only = false
export_dtype = "float32"
async_mode = "disabled" # ["disabled", "async", "async_with_pinned_mem"]

[activation_checkpoint]
mode = 'full'  # ['none', 'selective', 'full']

[float8]
enable_float8_linear = false