Update some config default values (#711)

Author: ebsmothers

recipes/configs/llama2/13B_lora.yaml

@@ -70,6 +70,7 @@ loss:
 # Training
 epochs: 1
 max_steps_per_epoch: null
+gradient_accumulation_steps: 1
 
 # Logging
 output_dir: /tmp/lora_finetune_output

recipes/configs/llama2/7B_lora.yaml

@@ -5,13 +5,13 @@
 # this run:
 #   tune download meta-llama/Llama-2-7b --hf-token <HF_TOKEN> --output-dir /tmp/llama2
 #
-# To launch on 4 devices, run the following command from root:
-#   tune run --nnodes 1 --nproc_per_node 4 lora_finetune_distributed --config llama2/7B_lora
+# To launch on 2 devices, run the following command from root:
+#   tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed --config llama2/7B_lora
 #
 # You can add specific overrides through the command line. For example
 # to override the checkpointer directory while launching training
 # you can run:
-#   tune run --nnodes 1 --nproc_per_node 4 lora_finetune_distributed --config llama2/7B_lora checkpointer.checkpoint_dir=<YOUR_CHECKPOINT_DIR>
+#   tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed --config llama2/7B_lora checkpointer.checkpoint_dir=<YOUR_CHECKPOINT_DIR>
 #
 # This config works best when the model is being fine-tuned on 2+ GPUs.
 # For single device LoRA finetuning please use 7B_lora_single_device.yaml
@@ -67,7 +67,7 @@ loss:
 # Training
 epochs: 1
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 32
 
 # Logging
 output_dir: /tmp/lora_finetune_output

recipes/configs/llama2/7B_lora_single_device.yaml

@@ -65,7 +65,7 @@ loss:
 # Training
 epochs: 1
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 64
 compile: False
 
 # Logging

recipes/configs/llama2/7B_qlora_single_device.yaml

@@ -64,7 +64,7 @@ loss:
 # Training
 epochs: 1
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 16
 # Note: compile for QLoRA is only supported on nightly
 # PyTorch (>= 2.4.0.dev20240408)
 compile: False

recipes/configs/mistral/7B_full_low_memory.yaml

@@ -56,7 +56,7 @@ optimizer:
 loss:
   _component_: torch.nn.CrossEntropyLoss
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 4
 optimizer_in_bwd: True
 
 # Training env

recipes/configs/mistral/7B_lora.yaml

@@ -9,13 +9,13 @@
 # this run:
 #   tune download mistralai/Mistral-7B-v0.1 --hf-token <HF_TOKEN> --output-dir /tmp/Mistral-7B-v0.1
 #
-# Run this config on 4 GPUs using the following:
-#   tune run --nnodes 1 --nproc_per_node 4 lora_finetune_distributed --config mistral/7B_lora
+# Run this config on 2 GPUs using the following:
+#   tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed --config mistral/7B_lora
 #
 # You can add specific overrides through the command line. For example
 # to override the checkpointer directory while launching training
 # you can run:
-#   tune run --nnodes 1 --nproc_per_node 4 lora_finetune_distributed --config mistral/7B_lora checkpointer.checkpoint_dir=<YOUR_CHECKPOINT_DIR>
+#   tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed --config mistral/7B_lora checkpointer.checkpoint_dir=<YOUR_CHECKPOINT_DIR>
 #
 # This config works best when the model is being fine-tuned on 2+ GPUs.
 # For single device LoRA finetuning please use 7B_lora_single_device.yaml

recipes/configs/mistral/7B_lora_single_device.yaml

@@ -67,7 +67,7 @@ loss:
 batch_size: 4
 epochs: 3
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 4
 compile: False
 
 # Training env

recipes/configs/mistral/7B_qlora_single_device.yaml

@@ -68,7 +68,7 @@ loss:
 batch_size: 4
 epochs: 3
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 4
 compile: False
 
 # Training env

tests/recipes/test_lora_finetune_distributed.py

@@ -41,6 +41,7 @@ def _get_test_config_overrides(self):
             "max_steps_per_epoch=2",
             "optimizer.lr=2e-5",
             "log_every_n_steps=1",
+            "gradient_accumulation_steps=1",
         ] + dummy_alpaca_dataset_config()
 
     def _fetch_expected_loss_values(self):

tests/recipes/test_lora_finetune_single_device.py

@@ -42,6 +42,7 @@ def _get_test_config_overrides(self, dtype_str: str = "fp32"):
             "max_steps_per_epoch=2",
             "optimizer.lr=2e-5",
             "log_every_n_steps=1",
+            "gradient_accumulation_steps=1",
         ] + dummy_alpaca_dataset_config()
 
     def _fetch_expected_loss_values(self):