Instella/configs/instella-3b-sft.yaml at main · AMD-AGI/Instella · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
run_name: instella-3b-sft
seed: 6198
dry_run: false

wandb:
  name: ${run_name}
  project: instella
  group: sft

model:
  d_model: 2560
  n_heads: 32
  n_layers: 36
  mlp_hidden_size: 13824
  weight_tying: false
  alibi: false
  rope: true
  flash_attention: true
  attention_dropout: 0.0
  attention_layer_norm: true
  multi_query_attention: false
  include_bias: false
  block_type: sequential
  layer_norm_type: rms
  layer_norm_with_affine: true
  bias_for_layer_norm: false
  attention_layer_norm_with_affine: true
  activation_type: swiglu
  residual_dropout: 0.0
  embedding_dropout: 0.0
  max_sequence_length: 4096
  vocab_size: 50280
  embedding_size: 50304
  eos_token_id: 0
  pad_token_id: 1
  init_device: meta
  init_fn: mitchell

compile:
  fullgraph: false

optimizer:
  name: adamw
  learning_rate: 1.0e-5
  weight_decay: 0.1
  decay_norm_and_bias: true
  decay_embeddings: true
  eps: 1e-8
  betas:
  - 0.9
  - 0.95
  metrics_log_interval: 10

scheduler:
  name: linear_with_warmup
  t_warmup: 500
  alpha_f: 0.001


tokenizer:
  identifier: tokenizers/allenai_eleuther-ai-gpt-neox-20b-pii-special.json
  truncate_direction: right

save_folder: outputs/sft/${run_name}

save_overwrite: true
# Sharded checkpoints (best for restarts)
save_interval: 1000
save_num_checkpoints_to_keep: 9
# Unsharded checkpoints (for final storage)
save_interval_unsharded: 10000
save_num_unsharded_checkpoints_to_keep: -1

load_path: outputs/pretrain/instella-3b-pretrain/step13727-unsharded
reset_trainer_state: true
reset_optimizer_state: true

max_duration: 3ep
global_train_batch_size: 128 #
device_train_microbatch_size: 4

precision: amp_bf16

fsdp:
  wrapping_strategy: null
  precision: mixed
  sharding_strategy: FULL_SHARD

max_grad_norm: 1.0
max_grad_norm_ratio: null

speed_monitor:
  window_size: 20

normalize_by_label_mask: true

data:
  pad_direction: right
  num_workers: 32
  drop_last: true
  pin_memory: true
  prefetch_factor: 1
  persistent_workers: true
  timeout: 0
  generate_attention_mask: false
  paths:
      - datasets/sft/smoltalk/input_ids.npy
      - datasets/sft/openmathinstruct2_1M/input_ids.npy
      - datasets/sft/tulu3-if/input_ids.npy
      - datasets/sft/o1-journey-10x/input_ids.npy
      - datasets/sft/mmlu/input_ids.npy
  label_mask_paths:
      - datasets/sft/smoltalk/label_mask.npy
      - datasets/sft/openmathinstruct2_1M/label_mask.npy
      - datasets/sft/tulu3-if/label_mask.npy
      - datasets/sft/o1-journey-10x/label_mask.npy
      - datasets/sft/mmlu/label_mask.npy