diff --git a/.assets/performance_profiles.png b/.assets/performance_profiles.png index ec587313e..a8bf2a7b5 100644 Binary files a/.assets/performance_profiles.png and b/.assets/performance_profiles.png differ diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 681b77e96..04f035ddc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,8 +19,8 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=fastmri --framework=pytorch --global_batch_size=8 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_momentum.py --tuning_search_space=reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json - python tests/reference_algorithm_tests.py --workload=fastmri --framework=jax --global_batch_size=8 --submission_path=reference_algorithms/target_setting_algorithms/jax_momentum.py --tuning_search_space=reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=fastmri --framework=pytorch --global_batch_size=8 --submission_path=algorithms/target_setting_algorithms/pytorch_momentum.py --tuning_search_space=algorithms/target_setting_algorithms/fastmri/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=fastmri --framework=jax --global_batch_size=8 --submission_path=algorithms/target_setting_algorithms/jax_momentum.py --tuning_search_space=algorithms/target_setting_algorithms/fastmri/tuning_search_space.json wmt_jax: runs-on: ubuntu-latest steps: @@ -37,7 +37,7 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=wmt --framework=jax --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/jax_nadamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=wmt --framework=jax --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/jax_nadamw.py --tuning_search_space=algorithms/target_setting_algorithms/wmt/tuning_search_space.json wmt_pytorch: runs-on: ubuntu-latest steps: @@ -54,7 +54,7 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=wmt --framework=pytorch --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nadamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=wmt --framework=pytorch --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/pytorch_nadamw.py --tuning_search_space=algorithms/target_setting_algorithms/wmt/tuning_search_space.json imagenet_jax: runs-on: ubuntu-latest steps: @@ -71,8 +71,8 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=imagenet_vit --framework=jax --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json - python tests/reference_algorithm_tests.py --workload=imagenet_resnet --framework=jax --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/jax_momentum.py --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=imagenet_vit --framework=jax --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=imagenet_resnet --framework=jax --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/jax_momentum.py --tuning_search_space=algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json imagenet_pytorch: runs-on: ubuntu-latest steps: @@ -89,8 +89,8 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=imagenet_resnet --framework=pytorch --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_momentum.py --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json - python tests/reference_algorithm_tests.py --workload=imagenet_vit --framework=pytorch --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=imagenet_resnet --framework=pytorch --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/pytorch_momentum.py --tuning_search_space=algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=imagenet_vit --framework=pytorch --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json # uncomment when https://github.com/mlcommons/algorithmic-efficiency/issues/339 is resolved. criteo_jax: runs-on: ubuntu-latest @@ -108,7 +108,7 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=criteo1tb_test --framework=jax --global_batch_size=1 --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=criteo1tb_test --framework=jax --global_batch_size=1 --submission_path=algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json criteo_pytorch: runs-on: ubuntu-latest steps: @@ -125,7 +125,7 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=criteo1tb_test --framework=pytorch --global_batch_size=1 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=criteo1tb_test --framework=pytorch --global_batch_size=1 --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json speech_jax: runs-on: ubuntu-latest steps: @@ -142,8 +142,8 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=librispeech_conformer --framework=jax --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json - python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech --framework=jax --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=librispeech_conformer --framework=jax --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech --framework=jax --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/jax_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json speech_pytorch: runs-on: ubuntu-latest steps: @@ -160,8 +160,8 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech --framework=pytorch --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json - python tests/reference_algorithm_tests.py --workload=librispeech_conformer --framework=pytorch --global_batch_size=2 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech --framework=pytorch --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=librispeech_conformer --framework=pytorch --global_batch_size=2 --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py --tuning_search_space=algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json ogbg: runs-on: ubuntu-latest steps: @@ -178,8 +178,8 @@ jobs: pip install .[pytorch_cpu] pip install .[full] pip install -e . - python tests/reference_algorithm_tests.py --workload=ogbg --framework=pytorch --global_batch_size=8 --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nesterov.py --tuning_search_space=reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json - python tests/reference_algorithm_tests.py --workload=ogbg --framework=jax --global_batch_size=8 --submission_path=reference_algorithms/target_setting_algorithms/jax_nesterov.py --tuning_search_space=reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=ogbg --framework=pytorch --global_batch_size=8 --submission_path=algorithms/target_setting_algorithms/pytorch_nesterov.py --tuning_search_space=algorithms/target_setting_algorithms/ogbg/tuning_search_space.json + python tests/reference_algorithm_tests.py --workload=ogbg --framework=jax --global_batch_size=8 --submission_path=algorithms/target_setting_algorithms/jax_nesterov.py --tuning_search_space=algorithms/target_setting_algorithms/ogbg/tuning_search_space.json pytest-params: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 172cf6102..1820d17c6 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -44,7 +44,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d fastmri -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w fastmri -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d fastmri -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w fastmri -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs imagenet_resnet_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -53,7 +53,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d imagenet -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w imagenet_resnet -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d imagenet -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w imagenet_resnet -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs imagenet_vit_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -62,7 +62,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d imagenet -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w imagenet_vit -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d imagenet -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w imagenet_vit -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs ogbg_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -71,7 +71,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d ogbg -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w ogbg -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d ogbg -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w ogbg -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs criteo_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -80,7 +80,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d criteo1tb -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w criteo1tb -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d criteo1tb -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w criteo1tb -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs librispeech_conformer_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -89,7 +89,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d librispeech -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w librispeech_conformer -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d librispeech -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w librispeech_conformer -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs librispeech_deepspeech_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -98,7 +98,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d librispeech -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w librispeech_deepspeech -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d librispeech -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w librispeech_deepspeech -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs wmt_jax: runs-on: self-hosted needs: build_and_push_jax_docker_image @@ -107,7 +107,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d wmt -f jax -s reference_algorithms/paper_baselines/adamw/jax/submission.py -w wmt -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_jax_${{ github.head_ref || github.ref_name }} -d wmt -f jax -s algorithms/archived_paper_baselines/adamw/jax/submission.py -w wmt -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs fastmri_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -116,7 +116,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d fastmri -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w fastmri -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d fastmri -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w fastmri -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs imagenet_resnet_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -125,7 +125,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d imagenet -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w imagenet_resnet -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d imagenet -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w imagenet_resnet -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs imagenet_vit_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -134,7 +134,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d imagenet -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w imagenet_vit -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d imagenet -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w imagenet_vit -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs ogbg_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -143,7 +143,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d ogbg -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w ogbg -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d ogbg -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w ogbg -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs criteo_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -152,7 +152,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d criteo1tb -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w criteo1tb -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d criteo1tb -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w criteo1tb -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs exit $? librispeech_conformer_pytorch: runs-on: self-hosted @@ -162,7 +162,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d librispeech -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w librispeech_conformer -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d librispeech -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w librispeech_conformer -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs librispeech_deepspeech_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -171,7 +171,7 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d librispeech -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w librispeech_deepspeech -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d librispeech -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w librispeech_deepspeech -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs wmt_pytorch: runs-on: self-hosted needs: build_and_push_pytorch_docker_image @@ -180,4 +180,4 @@ jobs: - name: Run containerized workload run: | docker pull us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} - docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d wmt -f pytorch -s reference_algorithms/paper_baselines/adamw/pytorch/submission.py -w wmt -t reference_algorithms/paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs + docker run -v $HOME/data/:/data/ -v $HOME/experiment_runs/:/experiment_runs -v $HOME/experiment_runs/logs:/logs --gpus all --ipc=host us-central1-docker.pkg.dev/training-algorithms-external/mlcommons-docker-repo/algoperf_pytorch_${{ github.head_ref || github.ref_name }} -d wmt -f pytorch -s algorithms/archived_paper_baselines/adamw/pytorch/submission.py -w wmt -t algorithms/archived_paper_baselines/adamw/tuning_search_space.json -e tests/regression_tests/adamw -m 10 -c False -o True -r false --data_bucket mlcommons-data --logs_bucket mlcommons-runs --data_bucket mlcommons-data --logs_bucket mlcommons-runs diff --git a/README.md b/README.md index f9e3f00c3..f8e6763b4 100644 --- a/README.md +++ b/README.md @@ -2,78 +2,92 @@

-MLCommons Logo +MLCommons Logo

- Leaderboard • - Getting Started • - Submit • - Documentation • - Contributing • - Benchmark/Results Paper + 🏆 Leaderboard • + 🚀 Getting Started • + 📥 Submit • + 📖 Docs/Rules +
+ 📜 Benchmark Paper • + 📊 Results Paper

-[![CI Status](https://img.shields.io/github/actions/workflow/status/mlcommons/algorithmic-efficiency/CI.yml?style=flat-square&logo=github&label=CI)](https://github.com/mlcommons/algorithmic-efficiency/actions/workflows/CI.yml) -[![Linting Status](https://img.shields.io/github/actions/workflow/status/mlcommons/algorithmic-efficiency/linting.yml?style=flat-square&logo=github&label=Linting)](https://github.com/mlcommons/algorithmic-efficiency/actions/workflows/linting.yml) -[![Code Style Ruff](https://img.shields.io/badge/Code%20Style-Ruff-brightgreen?style=flat-square&logo=ruff)](https://github.com/astral-sh/ruff) -[![GitHub License](https://img.shields.io/github/license/mlcommons/algorithmic-efficiency?style=flat-square&label=License)](LICENSE.md) -[![Discord](https://dcbadge.limes.pink/api/server/5FPXK7SMt6?style=flat-square)](https://discord.gg/5FPXK7SMt6) +

+ Version + CI Status + Linting Status + Code Style Ruff + GitHub License + Discord +

--- -Unlike benchmarks that focus on model architecture or hardware, the AlgoPerf benchmark isolates the training algorithm itself, measuring how quickly it can achieve target performance levels on a fixed set of representative deep learning tasks. These tasks span various domains, including image classification, speech recognition, machine translation, and more, all running on standardized hardware (8x NVIDIA V100 GPUs). The benchmark includes 8 base workloads, which are fully specified. In addition there are definitions for "randomized" workloads, which are variations of the fixed workloads, which are designed to discourage overfitting. These randomized workloads were used for scoring the AlgPerf competition but will not be used for future scoring. +The MLCommons™ **AlgoPerf: Training Algorithms benchmark** is designed to find **training algorithms that can train neural networks faster** by rigorously measuring how quickly they reach a specific performance target across a diverse set of deep learning workloads. -Submissions are evaluated based on their "time-to-result", i.e., the wall-clock time it takes to reach predefined validation and test set performance targets on each workload. Submissions are scored under one of two different tuning rulesets. The [external tuning rule set](https://github.com/mlcommons/algorithmic-efficiency/blob/main/docs/DOCUMENTATION.md#external-tuning-ruleset) allows a limited amount of hyperparameter tuning (20 quasirandom trials) for each workload. The [self-tuning rule set](https://github.com/mlcommons/algorithmic-efficiency/blob/main/docs/DOCUMENTATION.md#self-tuning-ruleset) allows no external tuning, so any tuning is done "on-the-clock". For each submission, a single, overall benchmark score is computed by integrating its "performance profile" across all fixed workloads. The performance profile captures the relative training time of the submission to the best submission on each workload. Therefore the score of each submission is a function of other submissions in the submission pool. The higher the benchmark score, the better the submission's overall performance. +When training neural nets, practitioners face many critical yet often opaque decisions: What optimizer to choose? How should its learning rate be tuned? What learning rate schedule should be used? These choices can make or break training, yet the community has lacked a clear, standardized way to identify the state of the art. +Unlike benchmarks focused on hardware or model architecture, AlgoPerf isolates the **training algorithm** itself, which includes the optimizer, regularization, data selection, and hyperparameters like the learning rate schedule. By standardizing the benchmark process, AlgoPerf offers a meaningful apples-to-apples comparison of training algorithms and follows the following **key principles**: ---- +- 🎯 **Fixed Target, Model & Hardware:** Submitted training algorithms must train a set of [**fixed models**](/docs/DOCUMENTATION.md#workloads) to a pre-defined validation performance target as fast as possible. All submissions use the same model architecture and are run on the same [**standardized hardware**](/docs/DOCUMENTATION.md#benchmarking-hardware) (8x NVIDIA V100 GPUs). This isolates the training algorithm's performance and allows a fair apples-to-apples comparison. +- ⏱️ **Time-To-Result:** Submissions are evaluated based on the total wall-clock time required to reach the target, rewarding practical and efficient algorithms. +- 🧠 **Diverse Workloads:** The benchmark includes [**8 diverse deep learning workloads**](/docs/DOCUMENTATION.md#workloads) across domains like image classification, speech recognition, and machine translation. A submission's score is computed by aggregating its performance, using [**performance profiles**](/docs/DOCUMENTATION.md#benchmark-score-using-performance-profiles), across all workloads to ensure general-purpose algorithms. +- 📦 **Fully-Specified Algorithms:** Submissions must be complete procedures and thus hyperparameter tuning is treated as part of the algorithm. Submissions can either provide a search space for automated tuning ([**External tuning ruleset**](/docs/DOCUMENTATION.md#external-tuning-ruleset)) or be hyperparameter-free ([**Self-tuning ruleset**](/docs/DOCUMENTATION.md#self-tuning-ruleset)) with any tuning done automatically and "on the clock". This measures an algorithm's _total_ practical cost and provides practitioners with a complete method, eliminating the guesswork of how to apply it. -> This is the repository for the _AlgoPerf: Training Algorithms benchmark_ measuring neural network training speedups due to algorithmic improvements. -> It is developed by the [MLCommons Algorithms Working Group](https://mlcommons.org/en/groups/research-algorithms/). -> This repository holds the benchmark code, the benchmark's [**technical documentation**](/docs/DOCUMENTATION.md) and [**getting started guides**](/docs/GETTING_STARTED.md). For a detailed description of the benchmark design, see our [**introductory paper**](https://arxiv.org/abs/2306.07179), for the results of the inaugural competition see our [**results paper**](https://openreview.net/forum?id=CtM5xjRSfm). +> [!IMPORTANT] > -> **See our [AlgoPerf Leaderboard](https://github.com/mlcommons/submissions_algorithms) for the latest results of the benchmark and to submit your algorithm.** +> **We have moved to a rolling leaderboard!** +> We invite you to submit your algorithm for evaluation, see our [**How to Submit**](#how-to-submit) section and the [**submission repository**](https://github.com/mlcommons/submissions_algorithms). The working group will review your submission and, if selected, run it on our hardware and add your results to the official [**AlgoPerf Leaderboard**](https://github.com/mlcommons/submissions_algorithms). **Note: we are currently focusing our efforts on the self-tuning leaderboard to strengthen its competitiveness.** --- -> [!IMPORTANT] -> For future iterations of the AlgoPerf: Training Algorithms benchmark competition, we are switching to a rolling leaderboard, making a few changes to the competition rules, and also run all selected submissions on our hardware. **To submit your algorithm to the next iteration of the benchmark, please see our [How to Submit](#how-to-submit) section and the [submission repository](https://github.com/mlcommons/submissions_algorithms) which hosts the up to date AlgoPerf leaderboard.** - ## Table of Contents -- [Installation](#installation) - [Getting Started](#getting-started) + - [Installation](#installation) + - [Run a Workload](#run-a-workload) + - [Develop Your Algorithm](#develop-your-algorithm) - [How to Submit](#how-to-submit) - - [Technical Documentation of the Benchmark \& FAQs](#technical-documentation-of-the-benchmark--faqs) -- [Contributing](#contributing) +- [Rules, Documentation \& FAQ](#rules-documentation--faq) +- [Contributing \& Resources](#contributing--resources) +- [Releases \& Roadmap](#releases--roadmap) +- [Training Algorithm Collection](#training-algorithm-collection) +- [Citing Our Work](#citing-our-work) - [License](#license) -- [Paper and Citing the AlgoPerf Benchmark](#paper-and-citing-the-algoperf-benchmark) -## Installation +## Getting Started + +Follow these steps to run a baseline algorithm and start developing your own submission. +A more detailed guide can be found in the [**Getting Started**](/docs/GETTING_STARTED.md) document. +If you run into any issues, please feel free to contact us. +Either [**file an issue**](https://github.com/mlcommons/algorithmic-efficiency/issues), ask a question on [**our Discord**](https://discord.gg/5FPXK7SMt6) or [**join our weekly meetings**](https://mlcommons.org/en/groups/research-algorithms/). -> [!TIP] > **If you have any questions about the benchmark competition or you run into any issues, please feel free to contact us.** Either [file an issue](https://github.com/mlcommons/algorithmic-efficiency/issues), ask a question on [our Discord](https://discord.gg/5FPXK7SMt6) or [join our weekly meetings](https://mlcommons.org/en/groups/research-algorithms/). +### Installation -You can install this package and dependencies in a [Python virtual environment](/docs/GETTING_STARTED.md#python-virtual-environment) or use a [Docker/Singularity/Apptainer container](/docs/GETTING_STARTED.md#docker) (recommended). -We recommend using a Docker container (or alternatively, a Singularity/Apptainer container) to ensure a similar environment to our scoring and testing environments. -Both options are described in detail in the [**Getting Started**](/docs/GETTING_STARTED.md) document. +We recommend using the provided [**Docker container**](/docs/GETTING_STARTED.md#docker) to ensure a reproducible environment similar to our scoring environment. +Alternatively, you can install the package and its dependencies in a Python virtual environment. +Both options are described in more detail in the [**Getting Started**](/docs/GETTING_STARTED.md) document. -*TL;DR to install the Jax version for GPU and all workload dependencies run:* +_TL;DR Install JAX version for GPU (with workload dependencies):_ ```bash pip3 install -e '.[pytorch_cpu,jax_gpu,full]' --extra-index-url https://download.pytorch.org/whl/cpu ``` -*TL;DR to install the PyTorch version for GPU and all workload dependencies run:* +_TL;DR Install PyTorch version for GPU (with workload dependencies):_ ```bash pip3 install -e '.[jax_cpu,pytorch_gpu,full]' ``` -## Getting Started +### Run a Workload -For detailed instructions on developing your own algorithm in the benchmark see the [Getting Started](/docs/GETTING_STARTED.md) document. +Use the `submission_runner.py` to run an experiment, i.e., train a workload using a specific training algorithm. +Here's how to run the AdamW baseline on the `mnist` workload. -_TL;DR running a JAX workload:_ +_TL;DR: Running a JAX workload:_ ```bash python3 submission_runner.py \ @@ -81,11 +95,11 @@ python3 submission_runner.py \ --workload=mnist \ --experiment_dir=$HOME/experiments \ --experiment_name=my_first_experiment \ - --submission_path=reference_algorithms/paper_baselines/adamw/jax/submission.py \ - --tuning_search_space=reference_algorithms/paper_baselines/adamw/tuning_search_space.json + --submission_path=algorithms/archived_paper_baselines/adamw/jax/submission.py \ + --tuning_search_space=algorithms/archived_paper_baselines/adamw/tuning_search_space.json ``` -_TL;DR running a PyTorch workload:_ +_TL;DR: Running a PyTorch workload:_ ```bash python3 submission_runner.py \ @@ -93,33 +107,81 @@ python3 submission_runner.py \ --workload=mnist \ --experiment_dir=$HOME/experiments \ --experiment_name=my_first_experiment \ - --submission_path=reference_algorithms/paper_baselines/adamw/pytorch/submission.py \ - --tuning_search_space=reference_algorithms/paper_baselines/adamw/tuning_search_space.json + --submission_path=algorithms/archived_paper_baselines/adamw/pytorch/submission.py \ + --tuning_search_space=algorithms/archived_paper_baselines/adamw/tuning_search_space.json ``` +### Develop Your Algorithm + +Now you're ready to create your own `submission.py`! For detailed instructions, FAQs, and technical details, please refer to our documentation: + +- [**Getting Started Guide**](/docs/GETTING_STARTED.md): A detailed walkthrough for developing your algorithm. +- [**Benchmark Documentation**](/docs/DOCUMENTATION.md): The complete technical reference including the "benchmark rules" such as allowed and disallowed submissions, FAQs, and technical details such as the API. + ## How to Submit -Once you have developed your training algorithm, you can submit it to the benchmark by creating a pull request to the [submission repository](https://github.com/mlcommons/submissions_algorithms), which hosts the AlgoPerf leaderboard. The AlgoPerf working group will review your PR. Based on our available resources and the perceived potential of the method, it will be selected for a free evaluation. If selected, we will run your algorithm on our hardware and update the leaderboard with the results. +Ready to see how your algorithm stacks up? Submit it to the official AlgoPerf leaderboard! -### Technical Documentation of the Benchmark & FAQs +1. **Develop Your Algorithm:** Create your training algorithm following the API and "rules" described in our [**documentation**](/docs/DOCUMENTATION.md). +2. **Create a Pull Request:** Fork the [**submissions repository**](https://github.com/mlcommons/submissions_algorithms) and create a pull request with your algorithm. +3. **Review and Evaluation:** The MLCommons Algorithms Working Group will review your PR. Based on its potential and our available resources, it may be selected for a **free, official evaluation** on our hardware. +4. **See Your Results:** If selected, we will run your algorithm and add the results to the [**public leaderboard**](https://github.com/mlcommons/submissions_algorithms). -We provide a technical documentation of the benchmark and answer frequently asked questions in a separate [**Documentation**](/docs/DOCUMENTATION.md) page. This includes which types of submissions are allowed. Please ensure that your submission is compliant with these rules before submitting. Suggestions, clarifications and questions can be raised via pull requests, creating an issue, or by sending an email to the [working group](mailto:algorithms@mlcommons.org). +## Rules, Documentation & FAQ -## Contributing +We provide a technical documentation of the benchmark and answer frequently asked questions regarding the benchmarking protocol in a dedicated [**Documentation**](/docs/DOCUMENTATION.md) page. This includes which types of submissions are allowed, a description of the benchmark API, and the entire benchmarking protocol. Please ensure that your submission is compliant with these rules before submitting. Suggestions, clarifications, and questions can be raised via pull requests, by creating an issue, or by reaching out to the [**working group**](mailto:algorithms@mlcommons.org). -We invite everyone to look through our rules, documentation, and codebase and submit issues and pull requests, e.g. for rules changes, clarifications, or any bugs you might encounter. If you are interested in contributing to the work of the working group and influence the benchmark's design decisions, please [join the weekly meetings](https://mlcommons.org/en/groups/research-algorithms/) and consider becoming a member of the working group. +For a detailed description and motivation of the initial benchmark design, please refer to our [**Benchmark Paper**](/docs/DOCUMENTATION.md#benchmark-paper). +For the results of the first AlgoPerf competition, please refer to our [**Competition Results Paper**](/docs/DOCUMENTATION.md#competition-results-paper). +See our [**AlgoPerf Leaderboard**](https://github.com/mlcommons/submissions_algorithms) for the latest results of the benchmark and the option to submit your algorithm. -Our [**Contributing**](/docs/CONTRIBUTING.md) document provides further MLCommons contributing guidelines and additional setup and workflow instructions. +## Contributing & Resources -## License +AlgoPerf is an open, community-driven project organized by the [MLCommons Algorithms Working Group](https://mlcommons.org/en/groups/research-algorithms/). Whether you want to submit an algorithm, report a bug, or help shape the future of the benchmark, we welcome your contributions. + +- 🏆 **Submit Your Algorithm:** Ready to compete? Create a pull request in the [**Submissions Repository**](https://github.com/mlcommons/submissions_algorithms). +- 🐞 **Report a Bug:** Found an issue with the codebase? Please [**file an issue**](https://github.com/mlcommons/algorithmic-efficiency/issues) so we can take a look. This also includes any rules changes or clarifications you would like to see. +- 🛠️ **Contribute to the Codebase:** We actively welcome pull requests! If you're interested in implementing new workloads, adding baselines, or fixing bugs please reach out to us. Our [**Contributing Guide**](/docs/CONTRIBUTING.md) offers further contributing guidelines and additional setup and workflow instructions. +- 👥 **Influence the Benchmark:** To contribute to the benchmark's design and direction, please join the [**weekly working group meetings**](https://mlcommons.org/en/groups/research-algorithms/). +- 💬 **Ask a Question:** Have a question or want to discuss ideas? Join the conversation on our [**Discord Server**](https://discord.gg/5FPXK7SMt6) or [**join our weekly meetings**](https://mlcommons.org/en/groups/research-algorithms/). + +## Releases & Roadmap + +The AlgoPerf benchmark is an actively evolving project designed to keep pace with the rapidly changing field of machine learning. To ensure clarity and reproducibility, we have adopted a unified versioning system: codebase, rules, and leaderboard all share the same `Major.Minor` version. `Patch` versions may differ for minor updates. +All results produced under the same `Major.Minor` version are comparable, making it easy to cite "`AlgoPerf v0.X`" and know exactly which set of rules, code, and submissions are being referenced. -The _AlgoPerf_ codebase is licensed under the [Apache License 2.0](/LICENSE.md). +Here is an overview of our key releases and the future roadmap. For a detailed list of changes in each release, see our [**Changelog**](docs/CHANGELOG.md). -## Paper and Citing the AlgoPerf Benchmark +- `v0.5` - Inaugural Competition
The benchmark as it was run for the first AlgoPerf competition in 2024. The key findings and analysis from this competition are detailed in our [**ICLR 2025 Results Paper**](https://openreview.net/forum?id=CtM5xjRSfm). It serves as a historical reference. + - **Leaderboard:** Archived at [**AlgoPerf v0.5 Leaderboard**](https://github.com/mlcommons/submissions_algorithms/tree/main/previous_leaderboards/algoperf_v05). + - **Rules:** The rules are archived at the [**AlgoPerf v0.5 Documentation**](https://github.com/mlcommons/algorithmic-efficiency/blob/v0.5.0/DOCUMENTATION.md). +- `v0.6` - **Current Version**
The active and recommended version of the benchmark. It is an improved and streamlined version that fixes important bugs and modifying the benchmarking protocol based on the lessons learned from the competition. **This is the recommended version for all new submissions.** -In our paper ["Benchmarking Neural Network Training Algorithms"](http://arxiv.org/abs/2306.07179) we motivate, describe, and justify the _AlgoPerf: Training Algorithms_ benchmark. + - **Key Changes:** (see the [Changelog](/docs/CHANGELOG.md) for details, including links to discussions on rule changes.) + - A rolling leaderboard now allows for continuous submissions and updates. + - Reduced computational cost via removing held-out workloads, 3 repetition studies (down from 5), and adjusted runtime budgets. + - Includes important bug fixes (e.g., batch norm) and API improvements (e.g., `prepare_for_eval` function). + - Migrating from `pmap` to `jit` in JAX for better performance and scalability. + - **Leaderboard:** The active (but currently limited) leaderboard can be found at [**AlgoPerf v0.6 Leaderboard**](https://github.com/mlcommons/submissions_algorithms). + - **Rules:** For the current set of rules see [**AlgoPerf v0.6 Documentation**](/docs/DOCUMENTATION.md). -If you are using the _AlgoPerf benchmark_, its codebase, baselines, or workloads, please consider citing our paper: +> 🏗️ `v1.0` (Future) - Planned Long-Term Support Release
This will be the next major release of the benchmark and a "long-term support" version, with the following **anticipated features:** +> +> - Adding a new language model (LM) workload. +> - Stronger baselines, especially for the self-tuning leaderboard. + +## Training Algorithm Collection + +This repository also provides a collection of implemented training algorithms with different purposes. These include [**submission templates**](./algorithms/template), [**development examples**](./algorithms/development_algorithms), [**target-setting algorithms**](./algorithms/target_setting_algorithms), [**historical baselines**](./algorithms/archived_paper_baselines), and [**current baselines**](./algorithms/baselines). For a detailed overview of these algorithms and their organization, please refer to the [`algorithms/README.md`](./algorithms/README.md) file. You can also find all benchmark submissions and their results on the official [**Leaderboard**](https://github.com/mlcommons/submissions_algorithms). +These algorithms provide a starting point for developing your own training algorithm and are a great resource for understanding the AlgoPerf benchmark and its API. + +## Citing Our Work + +If you use the AlgoPerf benchmark, its codebase, or results in your research, please cite our papers. + +**Benchmark Paper:** + +In this paper, we motivate, describe, and justify the _AlgoPerf: Training Algorithms_ benchmark. > [Dahl, Schneider, Nado, et al.
> **Benchmarking Neural Network Training Algorithms**
> _arXiv 2306.07179_](http://arxiv.org/abs/2306.07179) @@ -133,10 +195,11 @@ If you are using the _AlgoPerf benchmark_, its codebase, baselines, or workloads } ``` -If you use the results from the first _AlgoPerf competition_, please consider citing the results paper, as well as the relevant submissions: +**Competition Results Paper:** + +In this paper, we analyze the results of the first AlgoPerf competition. -> [Kasimbeg, Schneider, Eschenhagen, et al.
> **Accelerating neural network training: An analysis of the AlgoPerf competition**
-> ICLR 2025](https://openreview.net/forum?id=CtM5xjRSfm) +> [Kasimbeg, Schneider, Eschenhagen, et al.
> **Accelerating neural network training: An analysis of the AlgoPerf competition**
> _ICLR 2025_](https://openreview.net/forum?id=CtM5xjRSfm) ```bibtex @inproceedings{Kasimbeg2025AlgoPerfResults, @@ -147,3 +210,13 @@ year = {2025}, url = {https://openreview.net/forum?id=CtM5xjRSfm} } ``` + +## License + +The _AlgoPerf_ codebase is licensed under the [**Apache License 2.0**](/LICENSE.md). All AlgoPerf benchmark submissions must likewise be open-source under the same [**Apache License 2.0**](https://www.apache.org/licenses/LICENSE-2.0). + +--- + +

+MLCommons™ Algorithms Working Group • Join us! +

diff --git a/algorithms/README.md b/algorithms/README.md new file mode 100644 index 000000000..58508da68 --- /dev/null +++ b/algorithms/README.md @@ -0,0 +1,92 @@ +# Training Algorithms Collection + +This directory contains different training algorithms used in the project. They range from templates to (historic and current) baselines and algorithms used for target-setting. +Please also see our [Leaderboard](https://github.com/mlcommons/submissions_algorithms/tree/main) which contains all current and previous submissions. + +## Algorithm Groups + +The algorithms in this subdirectory are organized into the following groups: + +### 🧩 [`template`](./template) + +This directory provides a `submission.py` template for creating a new AlgoPerf submission. It includes placeholders for all required submission functions. To create a new algorithm, copy the `template/` directory and modify the placeholder implementations with your own. + +The submission functions are: + +- [`get_batch_size`](./template/submission.py#L13): Returns the batch size for a given workload. +- [`init_optimizer_state`](./template/submission.py#L32): Initializes the optimizer state. +- [`update_params`](./template/submission.py#L56): Updates the model parameters. +- [`prepare_for_eval`](./template/submission.py#L100): Prepares the model for evaluation. +- [`data_selection`](./template/submission.py#L140): Selects a batch of data from the input queue. + +### ⚙️ [`development_algorithms`](./development_algorithms) + +This directory contains two example algorithms for the `CIFAR-10` and `MNIST` workloads. While these workloads are not part of the official AlgoPerf benchmark, they serve as valuable tools for development and debugging. You can use them to: + +- **Verify your setup:** Quickly train a model on `MNIST` or `CIFAR-10` to confirm that your `algoperf` environment is installed and functioning correctly. +- **Understand the submission format:** The provided examples serve as practical templates for new submissions. + - The `MNIST` algorithm demonstrates a simple _Adam optimizer with a constant learning rate_. + - The `CIFAR-10` algorithm implements _SGD with a linear warmup and cosine decay learning rate schedule_. + +Both algorithms are available in JAX and PyTorch and include corresponding `tuning_search_space.json` files. The `MNIST` algorithm provides both a discrete and a continuous tuning search space. + +### 🗃️ [`archived_paper_baselines`](./archived_paper_baselines) + +This subdirectory contains the archived baseline submissions for the [external tuning ruleset](../README.md#external-tuning-ruleset), as presented in our paper, ["Benchmarking Neural Network Training Algorithms"](https://arxiv.org/abs/2306.07179). These baselines cover some of the most popular training algorithms, which is why we tested them in our original paper. They can now be used as templates or starting points for future submissions. These baselines are preserved exactly as they were run for our paper experiments and are not intended to be updated. The paper baselines are based on eight different update rules: + +- [AdamW](./archived_paper_baselines/adamw) +- [NadamW](./archived_paper_baselines/nadamw) +- [Heavy Ball (SGD with Momentum)](./archived_paper_baselines/momentum) +- [Nesterov (SGD with Nesterov Momentum)](./archived_paper_baselines/nesterov) +- [LAMB](./archived_paper_baselines/lamb) +- [Adafactor](./archived_paper_baselines/adafactor) +- [SAM (with Adam)](./archived_paper_baselines/sam) +- [Shampoo](./archived_paper_baselines/shampoo) + +For each update rule, we provide two distinct tuning search spaces: + +1. A space where the first momentum parameter (commonly denoted as $\beta_1$) is tuned. +2. A space where the first momentum parameter is fixed. + +All paper baselines are implemented in both PyTorch and JAX. + +### 📏 [`baselines`](./baselines) + +This directory contains the our baseline training algorithm for the AlgoPerf benchmark. +It uses NadamW with a linear warmup and cosine decay learning rate schedule. +There is both a [self-tuning version](./baselines/self_tuning) and an [externally tuned version](./baselines/external_tuning) of this algorithm. +The baseline is implemented in both JAX and PyTorch. + +For comparison, we also provide the training logs for the JAX baseline runs in the [`baselines/logs`](./baselines/logs) directory for both rulesets. +For benchmark results of the baseline, see our [Leaderboard](https://github.com/mlcommons/submissions_algorithms/tree/main). + +### 🎯 [`target_setting_algorithms`](./target_setting_algorithms) + +These algorithms were used to set the target metric values for the workloads. + +The procedure was as follows: + +1. For each workload, we ran four standard algorithms: AdamW, NadamW, Nesterov Momentum, and Heavy Ball Momentum. +2. For each workload, we then tuned the hyperparameters of all four algorithms using 200 trials using relatively broad search spaces (see Table 8 in [our benchmark paper](https://arxiv.org/abs/2306.07179)). +3. For each workload, we selected the best performing run (combination of algorithm and hyperparameters) across all runs. This configuration is reproduced here as the "target-setting algorithm" for this workload. +4. To finalize the targets, we retrained the workload 20 times, with different random seeds, using this target-setting algorithm. The final validation targets were the median values over these 20 runs, while the test targets were the worst-case test set performance achieved across those 10 repetitions that hit the validation target. + +The specific algorithm used for each workload is listed below, the corresponding `tuning_search_space.json` file provides the specific hyperparameter settings used for each workload: + +- **Criteo:** `NAdamW` +- **FastMRI:** `Nesterov` +- **ImageNet-Resnet:** `Heavy-ball Momentum` +- **ImageNet-ViT:** `NAdamW` +- **Librispeech-Conformer:** `NAdamW` +- **Librispeech-Deepspeech:** `NAdamW` +- **OGBG:** `Nesterov` +- **WMT:** `NAdamW` + +> [!NOTE] +> These are not valid submissions because they use a different hyperparameter setting per workload. However, they are included to ensure the reproducibility of the target-setting process. + +## ✔️ Submission Checker + +The `submission_checker.py` script can be used to verify that a submission is valid. It checks that the submission file has all the required functions and that they have the correct signatures. It does not check that the algorithm is correct, only that it is a valid submission. + +For more information on the submission format, see the [documentation](/DOCUMENTATION.md). diff --git a/reference_algorithms/__init__.py b/algorithms/__init__.py similarity index 100% rename from reference_algorithms/__init__.py rename to algorithms/__init__.py diff --git a/algorithms/archived_paper_baselines/README.md b/algorithms/archived_paper_baselines/README.md new file mode 100644 index 000000000..1ac903339 --- /dev/null +++ b/algorithms/archived_paper_baselines/README.md @@ -0,0 +1,14 @@ +# Baseline Submissions from the "Benchmarking Neural Network Training Algorithms" Paper + +This directory contains the baseline submissions for the [external tuning ruleset](../README.md#external-tuning-ruleset) as presented in our paper [Benchmarking Neural Network Training Algorithms](https://arxiv.org/abs/2306.07179). They are based on eight different update rules: + +- [AdamW](./adamw) +- [NadamW](./nadamw) +- [Heavy Ball (SGD with Momentum)](./momentum) +- [Nesterov (SGD with Nesterov Momentum)](./nesterov) +- [LAMB](./lamb) +- [Adafactor](./adafactor) +- [SAM (with Adam)](./sam) +- [Shampoo](./shampoo) + +Each update rule has two different tuning search spaces, one where the first momentum parameter (often denoted $\beta_1$) is tuned and one where it is set to a fixed value. diff --git a/reference_algorithms/paper_baselines/__init__.py b/algorithms/archived_paper_baselines/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/__init__.py rename to algorithms/archived_paper_baselines/__init__.py diff --git a/reference_algorithms/paper_baselines/adamw/__init__.py b/algorithms/archived_paper_baselines/adafactor/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/adamw/__init__.py rename to algorithms/archived_paper_baselines/adafactor/__init__.py diff --git a/reference_algorithms/paper_baselines/adamw/jax/__init__.py b/algorithms/archived_paper_baselines/adafactor/jax/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/adamw/jax/__init__.py rename to algorithms/archived_paper_baselines/adafactor/jax/__init__.py diff --git a/reference_algorithms/paper_baselines/adafactor/jax/sharded_adafactor.py b/algorithms/archived_paper_baselines/adafactor/jax/sharded_adafactor.py similarity index 100% rename from reference_algorithms/paper_baselines/adafactor/jax/sharded_adafactor.py rename to algorithms/archived_paper_baselines/adafactor/jax/sharded_adafactor.py diff --git a/reference_algorithms/paper_baselines/adafactor/jax/submission.py b/algorithms/archived_paper_baselines/adafactor/jax/submission.py similarity index 98% rename from reference_algorithms/paper_baselines/adafactor/jax/submission.py rename to algorithms/archived_paper_baselines/adafactor/jax/submission.py index b373e4f73..b0e3c11a2 100644 --- a/reference_algorithms/paper_baselines/adafactor/jax/submission.py +++ b/algorithms/archived_paper_baselines/adafactor/jax/submission.py @@ -10,7 +10,7 @@ from jax import lax from algoperf import spec -from reference_algorithms.paper_baselines.adafactor.jax.sharded_adafactor import ( +from algorithms.archived_paper_baselines.adafactor.jax.sharded_adafactor import ( sharded_adafactor, ) diff --git a/reference_algorithms/paper_baselines/adamw/pytorch/__init__.py b/algorithms/archived_paper_baselines/adafactor/pytorch/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/adamw/pytorch/__init__.py rename to algorithms/archived_paper_baselines/adafactor/pytorch/__init__.py diff --git a/reference_algorithms/paper_baselines/adafactor/pytorch/submission.py b/algorithms/archived_paper_baselines/adafactor/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/adafactor/pytorch/submission.py rename to algorithms/archived_paper_baselines/adafactor/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/adafactor/tuning_search_space.json b/algorithms/archived_paper_baselines/adafactor/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/adafactor/tuning_search_space.json rename to algorithms/archived_paper_baselines/adafactor/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/adafactor/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/adafactor/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/adafactor/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/adafactor/tuning_search_space_no_beta1.json diff --git a/reference_algorithms/paper_baselines/momentum/__init__.py b/algorithms/archived_paper_baselines/adamw/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/momentum/__init__.py rename to algorithms/archived_paper_baselines/adamw/__init__.py diff --git a/reference_algorithms/paper_baselines/momentum/jax/__init__.py b/algorithms/archived_paper_baselines/adamw/jax/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/momentum/jax/__init__.py rename to algorithms/archived_paper_baselines/adamw/jax/__init__.py diff --git a/reference_algorithms/paper_baselines/adamw/jax/submission.py b/algorithms/archived_paper_baselines/adamw/jax/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/adamw/jax/submission.py rename to algorithms/archived_paper_baselines/adamw/jax/submission.py diff --git a/reference_algorithms/paper_baselines/momentum/pytorch/__init__.py b/algorithms/archived_paper_baselines/adamw/pytorch/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/momentum/pytorch/__init__.py rename to algorithms/archived_paper_baselines/adamw/pytorch/__init__.py diff --git a/reference_algorithms/paper_baselines/adamw/pytorch/submission.py b/algorithms/archived_paper_baselines/adamw/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/adamw/pytorch/submission.py rename to algorithms/archived_paper_baselines/adamw/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/adamw/tuning_search_space.json b/algorithms/archived_paper_baselines/adamw/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/adamw/tuning_search_space.json rename to algorithms/archived_paper_baselines/adamw/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/adamw/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/adamw/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/adamw/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/adamw/tuning_search_space_no_beta1.json diff --git a/reference_algorithms/paper_baselines/nadamw/__init__.py b/algorithms/archived_paper_baselines/lamb/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/__init__.py rename to algorithms/archived_paper_baselines/lamb/__init__.py diff --git a/reference_algorithms/paper_baselines/nadamw/jax/__init__.py b/algorithms/archived_paper_baselines/lamb/jax/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/jax/__init__.py rename to algorithms/archived_paper_baselines/lamb/jax/__init__.py diff --git a/reference_algorithms/paper_baselines/lamb/jax/submission.py b/algorithms/archived_paper_baselines/lamb/jax/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/lamb/jax/submission.py rename to algorithms/archived_paper_baselines/lamb/jax/submission.py diff --git a/reference_algorithms/paper_baselines/nadamw/pytorch/__init__.py b/algorithms/archived_paper_baselines/lamb/pytorch/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/pytorch/__init__.py rename to algorithms/archived_paper_baselines/lamb/pytorch/__init__.py diff --git a/reference_algorithms/paper_baselines/lamb/pytorch/submission.py b/algorithms/archived_paper_baselines/lamb/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/lamb/pytorch/submission.py rename to algorithms/archived_paper_baselines/lamb/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/lamb/tuning_search_space.json b/algorithms/archived_paper_baselines/lamb/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/lamb/tuning_search_space.json rename to algorithms/archived_paper_baselines/lamb/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/lamb/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/lamb/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/lamb/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/lamb/tuning_search_space_no_beta1.json diff --git a/reference_algorithms/paper_baselines/nesterov/__init__.py b/algorithms/archived_paper_baselines/momentum/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/__init__.py rename to algorithms/archived_paper_baselines/momentum/__init__.py diff --git a/reference_algorithms/paper_baselines/nesterov/jax/__init__.py b/algorithms/archived_paper_baselines/momentum/jax/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/jax/__init__.py rename to algorithms/archived_paper_baselines/momentum/jax/__init__.py diff --git a/reference_algorithms/paper_baselines/momentum/jax/submission.py b/algorithms/archived_paper_baselines/momentum/jax/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/momentum/jax/submission.py rename to algorithms/archived_paper_baselines/momentum/jax/submission.py diff --git a/reference_algorithms/paper_baselines/nesterov/pytorch/__init__.py b/algorithms/archived_paper_baselines/momentum/pytorch/__init__.py similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/pytorch/__init__.py rename to algorithms/archived_paper_baselines/momentum/pytorch/__init__.py diff --git a/reference_algorithms/paper_baselines/momentum/pytorch/submission.py b/algorithms/archived_paper_baselines/momentum/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/momentum/pytorch/submission.py rename to algorithms/archived_paper_baselines/momentum/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/momentum/tuning_search_space.json b/algorithms/archived_paper_baselines/momentum/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/momentum/tuning_search_space.json rename to algorithms/archived_paper_baselines/momentum/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/momentum/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/momentum/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/momentum/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/momentum/tuning_search_space_no_beta1.json diff --git a/algorithms/archived_paper_baselines/nadamw/__init__.py b/algorithms/archived_paper_baselines/nadamw/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/algorithms/archived_paper_baselines/nadamw/jax/__init__.py b/algorithms/archived_paper_baselines/nadamw/jax/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/nadamw/jax/submission.py b/algorithms/archived_paper_baselines/nadamw/jax/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/jax/submission.py rename to algorithms/archived_paper_baselines/nadamw/jax/submission.py diff --git a/algorithms/archived_paper_baselines/nadamw/pytorch/__init__.py b/algorithms/archived_paper_baselines/nadamw/pytorch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/nadamw/pytorch/submission.py b/algorithms/archived_paper_baselines/nadamw/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/pytorch/submission.py rename to algorithms/archived_paper_baselines/nadamw/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/nadamw/tuning_search_space.json b/algorithms/archived_paper_baselines/nadamw/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/tuning_search_space.json rename to algorithms/archived_paper_baselines/nadamw/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/nadamw/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/nadamw/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/nadamw/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/nadamw/tuning_search_space_no_beta1.json diff --git a/algorithms/archived_paper_baselines/nesterov/__init__.py b/algorithms/archived_paper_baselines/nesterov/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/algorithms/archived_paper_baselines/nesterov/jax/__init__.py b/algorithms/archived_paper_baselines/nesterov/jax/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/nesterov/jax/submission.py b/algorithms/archived_paper_baselines/nesterov/jax/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/jax/submission.py rename to algorithms/archived_paper_baselines/nesterov/jax/submission.py diff --git a/algorithms/archived_paper_baselines/nesterov/pytorch/__init__.py b/algorithms/archived_paper_baselines/nesterov/pytorch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/nesterov/pytorch/submission.py b/algorithms/archived_paper_baselines/nesterov/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/pytorch/submission.py rename to algorithms/archived_paper_baselines/nesterov/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/nesterov/tuning_search_space.json b/algorithms/archived_paper_baselines/nesterov/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/tuning_search_space.json rename to algorithms/archived_paper_baselines/nesterov/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/nesterov/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/nesterov/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/nesterov/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/nesterov/tuning_search_space_no_beta1.json diff --git a/algorithms/archived_paper_baselines/sam/__init__.py b/algorithms/archived_paper_baselines/sam/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/algorithms/archived_paper_baselines/sam/jax/__init__.py b/algorithms/archived_paper_baselines/sam/jax/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/sam/jax/submission.py b/algorithms/archived_paper_baselines/sam/jax/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/sam/jax/submission.py rename to algorithms/archived_paper_baselines/sam/jax/submission.py diff --git a/algorithms/archived_paper_baselines/sam/pytorch/__init__.py b/algorithms/archived_paper_baselines/sam/pytorch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/sam/pytorch/submission.py b/algorithms/archived_paper_baselines/sam/pytorch/submission.py similarity index 100% rename from reference_algorithms/paper_baselines/sam/pytorch/submission.py rename to algorithms/archived_paper_baselines/sam/pytorch/submission.py diff --git a/reference_algorithms/paper_baselines/sam/tuning_search_space.json b/algorithms/archived_paper_baselines/sam/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/sam/tuning_search_space.json rename to algorithms/archived_paper_baselines/sam/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/sam/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/sam/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/sam/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/sam/tuning_search_space_no_beta1.json diff --git a/algorithms/archived_paper_baselines/shampoo/__init__.py b/algorithms/archived_paper_baselines/shampoo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/algorithms/archived_paper_baselines/shampoo/jax/__init__.py b/algorithms/archived_paper_baselines/shampoo/jax/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/shampoo/jax/distributed_shampoo.py b/algorithms/archived_paper_baselines/shampoo/jax/distributed_shampoo.py similarity index 100% rename from reference_algorithms/paper_baselines/shampoo/jax/distributed_shampoo.py rename to algorithms/archived_paper_baselines/shampoo/jax/distributed_shampoo.py diff --git a/reference_algorithms/paper_baselines/shampoo/jax/submission.py b/algorithms/archived_paper_baselines/shampoo/jax/submission.py similarity index 98% rename from reference_algorithms/paper_baselines/shampoo/jax/submission.py rename to algorithms/archived_paper_baselines/shampoo/jax/submission.py index 358aaf48c..8233671bd 100644 --- a/reference_algorithms/paper_baselines/shampoo/jax/submission.py +++ b/algorithms/archived_paper_baselines/shampoo/jax/submission.py @@ -10,7 +10,7 @@ from jax import lax from algoperf import spec -from reference_algorithms.paper_baselines.shampoo.jax.distributed_shampoo import ( +from algorithms.archived_paper_baselines.shampoo.jax.distributed_shampoo import ( distributed_shampoo, ) diff --git a/algorithms/archived_paper_baselines/shampoo/pytorch/__init__.py b/algorithms/archived_paper_baselines/shampoo/pytorch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/paper_baselines/shampoo/tuning_search_space.json b/algorithms/archived_paper_baselines/shampoo/tuning_search_space.json similarity index 100% rename from reference_algorithms/paper_baselines/shampoo/tuning_search_space.json rename to algorithms/archived_paper_baselines/shampoo/tuning_search_space.json diff --git a/reference_algorithms/paper_baselines/shampoo/tuning_search_space_no_beta1.json b/algorithms/archived_paper_baselines/shampoo/tuning_search_space_no_beta1.json similarity index 100% rename from reference_algorithms/paper_baselines/shampoo/tuning_search_space_no_beta1.json rename to algorithms/archived_paper_baselines/shampoo/tuning_search_space_no_beta1.json diff --git a/algorithms/baselines/README.md b/algorithms/baselines/README.md new file mode 100644 index 000000000..96665bb38 --- /dev/null +++ b/algorithms/baselines/README.md @@ -0,0 +1,81 @@ +# Baselines + +This directory contains the our baseline training algorithm for the AlgoPerf benchmark. +It uses NadamW with a linear warmup and cosine decay learning rate schedule. +There is both a [self-tuning version](./baselines/self_tuning) and an [externally tuned version](./baselines/external_tuning) of this algorithm. +The baseline is implemented in both JAX and PyTorch. + +For comparison, we also provide the training logs for the JAX baseline runs in the [`baselines/logs`](./baselines/logs) directory for both rulesets. +For benchmark results of the baseline, see our [Leaderboard](https://github.com/mlcommons/submissions_algorithms/tree/main). + +## Externally Tuned Ruleset + +The baseline submission for **JAX**: + +- `algorithms/baselines/external_tuning/jax_nadamw_full_budget.py` + +Example command: + +```bash +python3 submission_runner.py \ + --framework=jax \ + --data_dir= \ + --experiment_dir= \ + --experiment_name= \ + --workload= \ + --submission_path=algorithms/baselines/external_tuning/jax_nadamw_target_setting.py \ + --tuning_search_space=algorithms/baselines/external_tuning/tuning_search_space.json +``` + +The baseline submission for **PyTorch**: + +- `algorithms/baselines/external_tuning/pytorch_nadamw_full_budget.py` + +Example command: + +```bash +torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ + --framework=pytorch \ + --data_dir= \ + --experiment_dir= \ + --experiment_name=t \ + --workload=\ + --submission_path=algorithms/baselines/external_tuning/pytorch_nadamw_target_setting.py \ + --tuning_search_space=algorithms/baselines/external_tuning/tuning_search_space.json +``` + +## Self-tuning Ruleset + +The baseline submission for **JAX**: + +- `algorithms/baselines/self_tuning/jax_nadamw_full_budget.py` + +Example command: + +```bash +python3 submission_runner.py \ + --framework=jax \ + --data_dir= \ + --experiment_dir= \ + --experiment_name= \ + --workload= \ + --submission_path=algorithms/baselines/self_tuning/jax_nadamw_target_setting.py \ + --tuning_ruleset=self +``` + +The baseline submission for **PyTorch**: + +- `algorithms/baselines/self_tuning/pytorch_nadamw_full_budget.py` + +Example command: + +```bash +torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ + --framework=pytorch \ + --data_dir= \ + --experiment_dir= \ + --experiment_name=t \ + --workload=\ + --submission_path=algorithms/baselines/self_tuning/pytorch_nadamw_target_setting.py \ + --tuning_ruleset=self +``` diff --git a/reference_algorithms/qualification_baselines/external_tuning/jax_nadamw_full_budget.py b/algorithms/baselines/external_tuning/jax_nadamw_full_budget.py similarity index 99% rename from reference_algorithms/qualification_baselines/external_tuning/jax_nadamw_full_budget.py rename to algorithms/baselines/external_tuning/jax_nadamw_full_budget.py index 9cc826642..0577cd4e0 100644 --- a/reference_algorithms/qualification_baselines/external_tuning/jax_nadamw_full_budget.py +++ b/algorithms/baselines/external_tuning/jax_nadamw_full_budget.py @@ -1,7 +1,5 @@ """Submission file for an NAdamW optimizer with warmup+cosine LR in Jax.""" -# isort: off -# We have to turn off isort here to resolve a conflict between isort and yapf. from typing import ( Any, Callable, @@ -13,8 +11,8 @@ Tuple, Union, ) -# isort: on +# isort: on import chex import jax import jax.numpy as jnp diff --git a/reference_algorithms/qualification_baselines/external_tuning/pytorch_nadamw_full_budget.py b/algorithms/baselines/external_tuning/pytorch_nadamw_full_budget.py similarity index 100% rename from reference_algorithms/qualification_baselines/external_tuning/pytorch_nadamw_full_budget.py rename to algorithms/baselines/external_tuning/pytorch_nadamw_full_budget.py diff --git a/reference_algorithms/qualification_baselines/external_tuning/tuning_search_space.json b/algorithms/baselines/external_tuning/tuning_search_space.json similarity index 100% rename from reference_algorithms/qualification_baselines/external_tuning/tuning_search_space.json rename to algorithms/baselines/external_tuning/tuning_search_space.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/criteo1tb_jax_02-13-2024-13-02-24.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/criteo1tb_jax_02-13-2024-13-02-24.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/criteo1tb_jax_02-13-2024-13-02-24.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/criteo1tb_jax_02-13-2024-13-02-24.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/criteo1tb_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/criteo1tb_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/fastmri_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/fastmri_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-01-13.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-01-13.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-01-13.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-01-13.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_resnet_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_resnet_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-40-46.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-40-46.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-40-46.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-40-46.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/imagenet_vit_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/imagenet_vit_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-01-22-23.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-01-22-23.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-01-22-23.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-01-22-23.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_conformer_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_conformer_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-10-2024-22-49-44.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-10-2024-22-49-44.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-10-2024-22-49-44.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-10-2024-22-49-44.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/ogbg_jax_02-04-2024-21-11-51.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/ogbg_jax_02-04-2024-21-11-51.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/ogbg_jax_02-04-2024-21-11-51.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/ogbg_jax_02-04-2024-21-11-51.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/ogbg_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/ogbg_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/wmt_jax_02-06-2024-10-57-53.log b/algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/wmt_jax_02-06-2024-10-57-53.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_0/wmt_jax/wmt_jax_02-06-2024-10-57-53.log rename to algorithms/baselines/logs/external_tuning/baseline/study_0/wmt_jax/wmt_jax_02-06-2024-10-57-53.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-28-38.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-28-38.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-28-38.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-28-38.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/criteo1tb_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/criteo1tb_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/fastmri_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/fastmri_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-48.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-48.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-48.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-48.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_resnet_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_resnet_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-42-02.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-42-02.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-42-02.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-42-02.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/imagenet_vit_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/imagenet_vit_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-48-34.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-48-34.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-48-34.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-48-34.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_conformer_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_conformer_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-20-46.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-20-46.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-20-46.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-20-46.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/ogbg_jax_02-04-2024-21-48-01.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/ogbg_jax_02-04-2024-21-48-01.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/ogbg_jax_02-04-2024-21-48-01.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/ogbg_jax_02-04-2024-21-48-01.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/ogbg_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/ogbg_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/wmt_jax_02-06-2024-11-39-02.log b/algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/wmt_jax_02-06-2024-11-39-02.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_1/wmt_jax/wmt_jax_02-06-2024-11-39-02.log rename to algorithms/baselines/logs/external_tuning/baseline/study_1/wmt_jax/wmt_jax_02-06-2024-11-39-02.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-24-50.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-24-50.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-24-50.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-24-50.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/criteo1tb_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/criteo1tb_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/fastmri_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/fastmri_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-57.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-57.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-57.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-02-57.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_resnet_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_resnet_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-48-02.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-48-02.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-48-02.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-13-48-02.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/imagenet_vit_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/imagenet_vit_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-44-53.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-44-53.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-44-53.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-44-53.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_conformer_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_conformer_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-46-57.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-46-57.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-46-57.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-00-46-57.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/ogbg_jax_02-04-2024-23-24-16.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/ogbg_jax_02-04-2024-23-24-16.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/ogbg_jax_02-04-2024-23-24-16.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/ogbg_jax_02-04-2024-23-24-16.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/ogbg_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/ogbg_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/wmt_jax_02-06-2024-13-15-17.log b/algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/wmt_jax_02-06-2024-13-15-17.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_2/wmt_jax/wmt_jax_02-06-2024-13-15-17.log rename to algorithms/baselines/logs/external_tuning/baseline/study_2/wmt_jax/wmt_jax_02-06-2024-13-15-17.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/criteo1tb_jax_02-17-2024-11-56-37.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/criteo1tb_jax_02-17-2024-11-56-37.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/criteo1tb_jax_02-17-2024-11-56-37.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/criteo1tb_jax_02-17-2024-11-56-37.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/criteo1tb_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/criteo1tb_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/fastmri_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/fastmri_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-03-07.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-03-07.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-03-07.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-26-2024-19-03-07.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-30-2024-20-48-52.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-30-2024-20-48-52.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-30-2024-20-48-52.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/imagenet_resnet_jax_01-30-2024-20-48-52.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_resnet_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_resnet_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/imagenet_vit_jax_01-26-2024-19-08-22.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/imagenet_vit_jax_01-26-2024-19-08-22.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/imagenet_vit_jax_01-26-2024-19-08-22.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/imagenet_vit_jax_01-26-2024-19-08-22.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/imagenet_vit_jax_02-03-2024-15-24-33.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/imagenet_vit_jax_02-03-2024-15-24-33.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/imagenet_vit_jax_02-03-2024-15-24-33.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/imagenet_vit_jax_02-03-2024-15-24-33.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/imagenet_vit_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/imagenet_vit_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/librispeech_conformer_jax_02-18-2024-00-16-19.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/librispeech_conformer_jax_02-18-2024-00-16-19.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/librispeech_conformer_jax_02-18-2024-00-16-19.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/librispeech_conformer_jax_02-18-2024-00-16-19.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_conformer_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_conformer_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-15-2024-02-44-09.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-15-2024-02-44-09.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-15-2024-02-44-09.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-15-2024-02-44-09.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/ogbg_jax_02-09-2024-01-06-01.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/ogbg_jax_02-09-2024-01-06-01.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/ogbg_jax_02-09-2024-01-06-01.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/ogbg_jax_02-09-2024-01-06-01.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/ogbg_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/ogbg_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/wmt_jax_02-10-2024-14-47-06.log b/algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/wmt_jax_02-10-2024-14-47-06.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_3/wmt_jax/wmt_jax_02-10-2024-14-47-06.log rename to algorithms/baselines/logs/external_tuning/baseline/study_3/wmt_jax/wmt_jax_02-10-2024-14-47-06.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-38-52.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-38-52.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-38-52.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/criteo1tb_jax_02-13-2024-14-38-52.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/criteo1tb_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/criteo1tb_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/fastmri_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/fastmri_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/imagenet_resnet_jax_01-27-2024-01-35-38.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/imagenet_resnet_jax_01-27-2024-01-35-38.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/imagenet_resnet_jax_01-27-2024-01-35-38.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/imagenet_resnet_jax_01-27-2024-01-35-38.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_resnet_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_resnet_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-19-56-30.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-19-56-30.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-19-56-30.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/imagenet_vit_jax_01-30-2024-19-56-30.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/imagenet_vit_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/imagenet_vit_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-58-52.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-58-52.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-58-52.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/librispeech_conformer_jax_02-14-2024-02-58-52.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_conformer_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_conformer_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-04-05-57.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-04-05-57.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-04-05-57.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_02-11-2024-04-05-57.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/ogbg_jax_02-05-2024-03-48-25.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/ogbg_jax_02-05-2024-03-48-25.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/ogbg_jax_02-05-2024-03-48-25.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/ogbg_jax_02-05-2024-03-48-25.log diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/ogbg_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/ogbg_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_1/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_1/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_1/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_1/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_2/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_2/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_2/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_2/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_2/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_2/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_2/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_2/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_2/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_2/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_2/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_2/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_3/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_3/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_3/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_3/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_3/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_3/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_3/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_3/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_3/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_3/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_3/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_3/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_4/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_4/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_4/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_4/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_4/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_4/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_4/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_4/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_4/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_4/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_4/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_4/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_5/flags_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_5/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_5/flags_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_5/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_5/hparams.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_5/hparams.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_5/hparams.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_5/hparams.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_5/meta_data_0.json b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_5/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/trial_5/meta_data_0.json rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/trial_5/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/wmt_jax_02-06-2024-17-34-24.log b/algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/wmt_jax_02-06-2024-17-34-24.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/external_tuning/full_budget/study_4/wmt_jax/wmt_jax_02-06-2024-17-34-24.log rename to algorithms/baselines/logs/external_tuning/baseline/study_4/wmt_jax/wmt_jax_02-06-2024-17-34-24.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-14-45.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-14-45.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-14-45.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-14-45.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/fastmri_jax/fastmri_jax_03-12-2024-03-47-57.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/fastmri_jax/fastmri_jax_03-12-2024-03-47-57.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/fastmri_jax/fastmri_jax_03-12-2024-03-47-57.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/fastmri_jax/fastmri_jax_03-12-2024-03-47-57.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-10-31.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-10-31.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-10-31.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-10-31.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-32-59.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-32-59.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-32-59.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-32-59.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-09-52.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-09-52.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-09-52.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-09-52.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-07-50.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-07-50.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-07-50.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-07-50.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/ogbg_jax/ogbg_jax_03-05-2024-10-01-36.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/ogbg_jax/ogbg_jax_03-05-2024-10-01-36.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/ogbg_jax/ogbg_jax_03-05-2024-10-01-36.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/ogbg_jax/ogbg_jax_03-05-2024-10-01-36.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_0/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_0/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/wmt_jax/wmt_jax_03-12-2024-04-28-16.log b/algorithms/baselines/logs/self_tuning/baseline/study_0/wmt_jax/wmt_jax_03-12-2024-04-28-16.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_0/wmt_jax/wmt_jax_03-12-2024-04-28-16.log rename to algorithms/baselines/logs/self_tuning/baseline/study_0/wmt_jax/wmt_jax_03-12-2024-04-28-16.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-39-31.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-39-31.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-39-31.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/criteo1tb_jax/criteo1tb_jax_03-13-2024-10-39-31.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/fastmri_jax/fastmri_jax_03-12-2024-03-32-44.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/fastmri_jax/fastmri_jax_03-12-2024-03-32-44.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/fastmri_jax/fastmri_jax_03-12-2024-03-32-44.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/fastmri_jax/fastmri_jax_03-12-2024-03-32-44.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-32-47.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-32-47.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-32-47.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-32-47.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-55-17.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-55-17.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-55-17.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-55-17.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-19-59-18.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-19-59-18.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-19-59-18.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-19-59-18.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-37-44.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-37-44.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-37-44.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-23-37-44.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/ogbg_jax/ogbg_jax_03-05-2024-10-18-35.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/ogbg_jax/ogbg_jax_03-05-2024-10-18-35.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/ogbg_jax/ogbg_jax_03-05-2024-10-18-35.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/ogbg_jax/ogbg_jax_03-05-2024-10-18-35.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_1/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_1/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/wmt_jax/wmt_jax_03-12-2024-04-28-04.log b/algorithms/baselines/logs/self_tuning/baseline/study_1/wmt_jax/wmt_jax_03-12-2024-04-28-04.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_1/wmt_jax/wmt_jax_03-12-2024-04-28-04.log rename to algorithms/baselines/logs/self_tuning/baseline/study_1/wmt_jax/wmt_jax_03-12-2024-04-28-04.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/criteo1tb_jax/criteo1tb_jax_03-13-2024-09-51-13.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/criteo1tb_jax/criteo1tb_jax_03-13-2024-09-51-13.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/criteo1tb_jax/criteo1tb_jax_03-13-2024-09-51-13.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/criteo1tb_jax/criteo1tb_jax_03-13-2024-09-51-13.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/fastmri_jax/fastmri_jax_03-12-2024-03-34-27.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/fastmri_jax/fastmri_jax_03-12-2024-03-34-27.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/fastmri_jax/fastmri_jax_03-12-2024-03-34-27.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/fastmri_jax/fastmri_jax_03-12-2024-03-34-27.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-39.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-39.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-39.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-39.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-06-03.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-06-03.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-06-03.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-06-03.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-18-55-53.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-18-55-53.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-18-55-53.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-18-55-53.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-22-49-22.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-22-49-22.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-22-49-22.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-12-2024-22-49-22.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/ogbg_jax/ogbg_jax_03-05-2024-10-04-38.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/ogbg_jax/ogbg_jax_03-05-2024-10-04-38.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/ogbg_jax/ogbg_jax_03-05-2024-10-04-38.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/ogbg_jax/ogbg_jax_03-05-2024-10-04-38.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_2/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_2/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/wmt_jax/wmt_jax_03-12-2024-04-14-45.log b/algorithms/baselines/logs/self_tuning/baseline/study_2/wmt_jax/wmt_jax_03-12-2024-04-14-45.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_2/wmt_jax/wmt_jax_03-12-2024-04-14-45.log rename to algorithms/baselines/logs/self_tuning/baseline/study_2/wmt_jax/wmt_jax_03-12-2024-04-14-45.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/criteo1tb_jax/criteo1tb_jax_03-16-2024-12-03-01.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/criteo1tb_jax/criteo1tb_jax_03-16-2024-12-03-01.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/criteo1tb_jax/criteo1tb_jax_03-16-2024-12-03-01.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/criteo1tb_jax/criteo1tb_jax_03-16-2024-12-03-01.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/fastmri_jax/fastmri_jax_03-12-2024-03-34-36.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/fastmri_jax/fastmri_jax_03-12-2024-03-34-36.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/fastmri_jax/fastmri_jax_03-12-2024-03-34-36.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/fastmri_jax/fastmri_jax_03-12-2024-03-34-36.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-57.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-57.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-57.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_resnet_jax/imagenet_resnet_jax_02-29-2024-05-13-57.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-31-55.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-31-55.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-31-55.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_vit_jax/imagenet_vit_jax_03-02-2024-11-31-55.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_conformer_jax/librispeech_conformer_jax_03-16-2024-21-22-50.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_conformer_jax/librispeech_conformer_jax_03-16-2024-21-22-50.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_conformer_jax/librispeech_conformer_jax_03-16-2024-21-22-50.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_conformer_jax/librispeech_conformer_jax_03-16-2024-21-22-50.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-15-2024-00-20-37.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-15-2024-00-20-37.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-15-2024-00-20-37.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-15-2024-00-20-37.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/ogbg_jax/ogbg_jax_03-05-2024-10-10-44.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/ogbg_jax/ogbg_jax_03-05-2024-10-10-44.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/ogbg_jax/ogbg_jax_03-05-2024-10-10-44.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/ogbg_jax/ogbg_jax_03-05-2024-10-10-44.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_3/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_3/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/wmt_jax/wmt_jax_03-12-2024-04-09-55.log b/algorithms/baselines/logs/self_tuning/baseline/study_3/wmt_jax/wmt_jax_03-12-2024-04-09-55.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_3/wmt_jax/wmt_jax_03-12-2024-04-09-55.log rename to algorithms/baselines/logs/self_tuning/baseline/study_3/wmt_jax/wmt_jax_03-12-2024-04-09-55.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/criteo1tb_jax/criteo1tb_jax_03-13-2024-12-00-36.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/criteo1tb_jax/criteo1tb_jax_03-13-2024-12-00-36.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/criteo1tb_jax/criteo1tb_jax_03-13-2024-12-00-36.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/criteo1tb_jax/criteo1tb_jax_03-13-2024-12-00-36.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/criteo1tb_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/criteo1tb_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/criteo1tb_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/criteo1tb_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/criteo1tb_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/criteo1tb_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/criteo1tb_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/criteo1tb_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/fastmri_jax/fastmri_jax_03-12-2024-03-27-32.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/fastmri_jax/fastmri_jax_03-12-2024-03-27-32.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/fastmri_jax/fastmri_jax_03-12-2024-03-27-32.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/fastmri_jax/fastmri_jax_03-12-2024-03-27-32.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/fastmri_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/fastmri_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/fastmri_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/fastmri_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/fastmri_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/fastmri_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/fastmri_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/fastmri_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_resnet_jax/imagenet_resnet_jax_03-05-2024-22-45-59.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_resnet_jax/imagenet_resnet_jax_03-05-2024-22-45-59.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_resnet_jax/imagenet_resnet_jax_03-05-2024-22-45-59.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_resnet_jax/imagenet_resnet_jax_03-05-2024-22-45-59.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_resnet_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_vit_jax/imagenet_vit_jax_03-08-2024-05-00-35.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_vit_jax/imagenet_vit_jax_03-08-2024-05-00-35.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_vit_jax/imagenet_vit_jax_03-08-2024-05-00-35.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_vit_jax/imagenet_vit_jax_03-08-2024-05-00-35.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_vit_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_vit_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_vit_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/imagenet_vit_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/imagenet_vit_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-34-38.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-34-38.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-34-38.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_conformer_jax/librispeech_conformer_jax_03-13-2024-20-34-38.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_conformer_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-13-2024-00-19-52.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-13-2024-00-19-52.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-13-2024-00-19-52.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_deepspeech_jax/librispeech_deepspeech_jax_03-13-2024-00-19-52.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/librispeech_deepspeech_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/ogbg_jax/ogbg_jax_03-11-2024-03-14-14.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/ogbg_jax/ogbg_jax_03-11-2024-03-14-14.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/ogbg_jax/ogbg_jax_03-11-2024-03-14-14.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/ogbg_jax/ogbg_jax_03-11-2024-03-14-14.log diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/ogbg_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/ogbg_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/ogbg_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/ogbg_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/ogbg_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/ogbg_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/ogbg_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/ogbg_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/wmt_jax/trial_1/flags_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/wmt_jax/trial_1/flags_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/wmt_jax/trial_1/flags_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/wmt_jax/trial_1/flags_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/wmt_jax/trial_1/meta_data_0.json b/algorithms/baselines/logs/self_tuning/baseline/study_4/wmt_jax/trial_1/meta_data_0.json similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/wmt_jax/trial_1/meta_data_0.json rename to algorithms/baselines/logs/self_tuning/baseline/study_4/wmt_jax/trial_1/meta_data_0.json diff --git a/reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/wmt_jax/wmt_jax_03-12-2024-04-02-58.log b/algorithms/baselines/logs/self_tuning/baseline/study_4/wmt_jax/wmt_jax_03-12-2024-04-02-58.log similarity index 100% rename from reference_algorithms/qualification_baselines/logs/self_tuning/full_budget/study_4/wmt_jax/wmt_jax_03-12-2024-04-02-58.log rename to algorithms/baselines/logs/self_tuning/baseline/study_4/wmt_jax/wmt_jax_03-12-2024-04-02-58.log diff --git a/reference_algorithms/qualification_baselines/self_tuning/jax_nadamw_full_budget.py b/algorithms/baselines/self_tuning/jax_nadamw_full_budget.py similarity index 100% rename from reference_algorithms/qualification_baselines/self_tuning/jax_nadamw_full_budget.py rename to algorithms/baselines/self_tuning/jax_nadamw_full_budget.py diff --git a/reference_algorithms/qualification_baselines/self_tuning/pytorch_nadamw_full_budget.py b/algorithms/baselines/self_tuning/pytorch_nadamw_full_budget.py similarity index 100% rename from reference_algorithms/qualification_baselines/self_tuning/pytorch_nadamw_full_budget.py rename to algorithms/baselines/self_tuning/pytorch_nadamw_full_budget.py diff --git a/algorithms/development_algorithms/__init__.py b/algorithms/development_algorithms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/algorithms/development_algorithms/cifar/__init__.py b/algorithms/development_algorithms/cifar/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/algorithms/development_algorithms/cifar/cifar_jax/__init__.py b/algorithms/development_algorithms/cifar/cifar_jax/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/development_algorithms/cifar/cifar_jax/submission.py b/algorithms/development_algorithms/cifar/cifar_jax/submission.py similarity index 100% rename from reference_algorithms/development_algorithms/cifar/cifar_jax/submission.py rename to algorithms/development_algorithms/cifar/cifar_jax/submission.py diff --git a/algorithms/development_algorithms/cifar/cifar_pytorch/__init__.py b/algorithms/development_algorithms/cifar/cifar_pytorch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/development_algorithms/cifar/cifar_pytorch/submission.py b/algorithms/development_algorithms/cifar/cifar_pytorch/submission.py similarity index 100% rename from reference_algorithms/development_algorithms/cifar/cifar_pytorch/submission.py rename to algorithms/development_algorithms/cifar/cifar_pytorch/submission.py diff --git a/reference_algorithms/development_algorithms/cifar/tuning_search_space.json b/algorithms/development_algorithms/cifar/tuning_search_space.json similarity index 100% rename from reference_algorithms/development_algorithms/cifar/tuning_search_space.json rename to algorithms/development_algorithms/cifar/tuning_search_space.json diff --git a/algorithms/development_algorithms/mnist/__init__.py b/algorithms/development_algorithms/mnist/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/development_algorithms/mnist/discrete_space.json b/algorithms/development_algorithms/mnist/discrete_space.json similarity index 100% rename from reference_algorithms/development_algorithms/mnist/discrete_space.json rename to algorithms/development_algorithms/mnist/discrete_space.json diff --git a/algorithms/development_algorithms/mnist/mnist_jax/__init__.py b/algorithms/development_algorithms/mnist/mnist_jax/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/development_algorithms/mnist/mnist_jax/submission.py b/algorithms/development_algorithms/mnist/mnist_jax/submission.py similarity index 100% rename from reference_algorithms/development_algorithms/mnist/mnist_jax/submission.py rename to algorithms/development_algorithms/mnist/mnist_jax/submission.py diff --git a/algorithms/development_algorithms/mnist/mnist_pytorch/__init__.py b/algorithms/development_algorithms/mnist/mnist_pytorch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/reference_algorithms/development_algorithms/mnist/mnist_pytorch/submission.py b/algorithms/development_algorithms/mnist/mnist_pytorch/submission.py similarity index 100% rename from reference_algorithms/development_algorithms/mnist/mnist_pytorch/submission.py rename to algorithms/development_algorithms/mnist/mnist_pytorch/submission.py diff --git a/reference_algorithms/development_algorithms/mnist/tuning_search_space.json b/algorithms/development_algorithms/mnist/tuning_search_space.json similarity index 100% rename from reference_algorithms/development_algorithms/mnist/tuning_search_space.json rename to algorithms/development_algorithms/mnist/tuning_search_space.json diff --git a/submissions/submission_checker.py b/algorithms/submission_checker.py similarity index 100% rename from submissions/submission_checker.py rename to algorithms/submission_checker.py diff --git a/reference_algorithms/target_setting_algorithms/README.md b/algorithms/target_setting_algorithms/README.md similarity index 61% rename from reference_algorithms/target_setting_algorithms/README.md rename to algorithms/target_setting_algorithms/README.md index 822907ba3..aee2b4cd5 100644 --- a/reference_algorithms/target_setting_algorithms/README.md +++ b/algorithms/target_setting_algorithms/README.md @@ -1,15 +1,19 @@ # Target Setting Run replications + Original runs were run on Google TPUv2-8 machines. These are not valid submissions, because they use a different hyperparameter setting per workload. But we include them in order to reproduce how we set the target metric values. To simplify directory setting, set: + ```bash ROOT_DIR=/home/znado ``` ## Criteo + Target was set using NAdamW with a linear warmup cosine decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -17,9 +21,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=criteo1tb \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -27,12 +32,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=criteo1tb \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json ``` ## FastMRI + Target was set using Nesterov with a linear warmup and linear decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -40,9 +47,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=fastmri \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_nesterov.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_nesterov.py \ + --tuning_search_space=algorithms/target_setting_algorithms/fastmri/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -50,12 +58,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=fastmri \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nesterov.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_nesterov.py \ + --tuning_search_space=algorithms/target_setting_algorithms/fastmri/tuning_search_space.json ``` ## ImageNet-Resnet + Target was set using Heavy-ball Momentum with a linear warmup and linear decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -64,9 +74,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=imagenet_resnet \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_momentum.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_momentum.py \ + --tuning_search_space=algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -75,12 +86,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=imagenet_resnet \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_momentum.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_momentum.py \ + --tuning_search_space=algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json ``` ## ImageNet-ViT + Target was set using NAdamW with a linear warmup cosine decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -89,9 +102,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=imagenet_vit \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -100,12 +114,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=imagenet_vit \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json ``` ## Librispeech-Conformer + Target was set using NAdamW with a linear warmup cosine decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -113,9 +129,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=librispeech_conformer \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_adamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -123,12 +140,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=librispeech_conformer \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json ``` ## Librispeech-Deepspeech + Target was set using NAdamW with a linear warmup cosine decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -136,9 +155,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=librispeech_deepspeech \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -146,12 +166,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=librispeech_deepspeech \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json ``` ## OGBG + Target was set using Nesterov with a linear warmup and linear decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -159,9 +181,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=ogbg \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_nesterov.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_nesterov.py \ + --tuning_search_space=algorithms/target_setting_algorithms/ogbg/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -169,12 +192,14 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=ogbg \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nesterov.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_nesterov.py \ + --tuning_search_space=algorithms/target_setting_algorithms/ogbg/tuning_search_space.json ``` ## WMT + Target was set using NAdamW with a linear warmup cosine decay LR schedule. + ```bash python3 submission_runner.py \ --framework=jax \ @@ -182,9 +207,10 @@ python3 submission_runner.py \ --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=wmt \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/wmt/tuning_search_space.json ``` + ```bash torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ --framework=pytorch \ @@ -192,6 +218,6 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc --experiment_dir=$ROOT_DIR \ --experiment_name=target_setting \ --workload=wmt \ - --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nadamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/pytorch_nadamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/wmt/tuning_search_space.json ``` diff --git a/reference_algorithms/target_setting_algorithms/__init__.py b/algorithms/target_setting_algorithms/__init__.py similarity index 100% rename from reference_algorithms/target_setting_algorithms/__init__.py rename to algorithms/target_setting_algorithms/__init__.py diff --git a/reference_algorithms/target_setting_algorithms/cosine_warmup.py b/algorithms/target_setting_algorithms/cosine_warmup.py similarity index 100% rename from reference_algorithms/target_setting_algorithms/cosine_warmup.py rename to algorithms/target_setting_algorithms/cosine_warmup.py diff --git a/reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json b/algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json rename to algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/criteo1tb_embed_init/tuning_search_space.json b/algorithms/target_setting_algorithms/criteo1tb_embed_init/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/criteo1tb_embed_init/tuning_search_space.json rename to algorithms/target_setting_algorithms/criteo1tb_embed_init/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/criteo1tb_layernorm/tuning_search_space.json b/algorithms/target_setting_algorithms/criteo1tb_layernorm/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/criteo1tb_layernorm/tuning_search_space.json rename to algorithms/target_setting_algorithms/criteo1tb_layernorm/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/criteo1tb_resnet/tuning_search_space.json b/algorithms/target_setting_algorithms/criteo1tb_resnet/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/criteo1tb_resnet/tuning_search_space.json rename to algorithms/target_setting_algorithms/criteo1tb_resnet/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/data_selection.py b/algorithms/target_setting_algorithms/data_selection.py similarity index 100% rename from reference_algorithms/target_setting_algorithms/data_selection.py rename to algorithms/target_setting_algorithms/data_selection.py diff --git a/reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json b/algorithms/target_setting_algorithms/fastmri/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json rename to algorithms/target_setting_algorithms/fastmri/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/fastmri_layernorm/tuning_search_space.json b/algorithms/target_setting_algorithms/fastmri_layernorm/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/fastmri_layernorm/tuning_search_space.json rename to algorithms/target_setting_algorithms/fastmri_layernorm/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/fastmri_model_size/tuning_search_space.json b/algorithms/target_setting_algorithms/fastmri_model_size/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/fastmri_model_size/tuning_search_space.json rename to algorithms/target_setting_algorithms/fastmri_model_size/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/fastmri_tanh/tuning_search_space.json b/algorithms/target_setting_algorithms/fastmri_tanh/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/fastmri_tanh/tuning_search_space.json rename to algorithms/target_setting_algorithms/fastmri_tanh/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/get_batch_size.py b/algorithms/target_setting_algorithms/get_batch_size.py similarity index 100% rename from reference_algorithms/target_setting_algorithms/get_batch_size.py rename to algorithms/target_setting_algorithms/get_batch_size.py diff --git a/reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_resnet_gelu/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_resnet_gelu/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_resnet_gelu/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_resnet_gelu/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_resnet_large_bn_init/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_resnet_large_bn_init/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_resnet_large_bn_init/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_resnet_large_bn_init/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_resnet_silu/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_resnet_silu/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_resnet_silu/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_resnet_silu/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_vit_glu/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_vit_glu/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_vit_glu/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_vit_glu/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_vit_map/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_vit_map/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_vit_map/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_vit_map/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/imagenet_vit_post_ln/tuning_search_space.json b/algorithms/target_setting_algorithms/imagenet_vit_post_ln/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/imagenet_vit_post_ln/tuning_search_space.json rename to algorithms/target_setting_algorithms/imagenet_vit_post_ln/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/jax_adamw.py b/algorithms/target_setting_algorithms/jax_adamw.py similarity index 77% rename from reference_algorithms/target_setting_algorithms/jax_adamw.py rename to algorithms/target_setting_algorithms/jax_adamw.py index 76b9fe6eb..0df08746a 100644 --- a/reference_algorithms/target_setting_algorithms/jax_adamw.py +++ b/algorithms/target_setting_algorithms/jax_adamw.py @@ -5,14 +5,14 @@ import optax from algoperf import spec -from reference_algorithms.target_setting_algorithms import cosine_warmup -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms import cosine_warmup +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/jax_momentum.py b/algorithms/target_setting_algorithms/jax_momentum.py similarity index 91% rename from reference_algorithms/target_setting_algorithms/jax_momentum.py rename to algorithms/target_setting_algorithms/jax_momentum.py index a9ed28442..950c010d6 100644 --- a/reference_algorithms/target_setting_algorithms/jax_momentum.py +++ b/algorithms/target_setting_algorithms/jax_momentum.py @@ -7,13 +7,13 @@ import optax from algoperf import spec -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/jax_nadamw.py b/algorithms/target_setting_algorithms/jax_nadamw.py similarity index 95% rename from reference_algorithms/target_setting_algorithms/jax_nadamw.py rename to algorithms/target_setting_algorithms/jax_nadamw.py index 68f1b5b3e..a172788ec 100644 --- a/reference_algorithms/target_setting_algorithms/jax_nadamw.py +++ b/algorithms/target_setting_algorithms/jax_nadamw.py @@ -8,14 +8,14 @@ import optax from algoperf import spec -from reference_algorithms.target_setting_algorithms import cosine_warmup -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms import cosine_warmup +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/jax_nesterov.py b/algorithms/target_setting_algorithms/jax_nesterov.py similarity index 91% rename from reference_algorithms/target_setting_algorithms/jax_nesterov.py rename to algorithms/target_setting_algorithms/jax_nesterov.py index bfa5c1755..961fa51cc 100644 --- a/reference_algorithms/target_setting_algorithms/jax_nesterov.py +++ b/algorithms/target_setting_algorithms/jax_nesterov.py @@ -7,13 +7,13 @@ import optax from algoperf import spec -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/jax_submission_base.py b/algorithms/target_setting_algorithms/jax_submission_base.py similarity index 100% rename from reference_algorithms/target_setting_algorithms/jax_submission_base.py rename to algorithms/target_setting_algorithms/jax_submission_base.py diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_no_resnet/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_no_resnet/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_no_resnet/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_no_resnet/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_deepspeech/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json b/algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json rename to algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json b/algorithms/target_setting_algorithms/ogbg/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json rename to algorithms/target_setting_algorithms/ogbg/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/ogbg_gelu/tuning_search_space.json b/algorithms/target_setting_algorithms/ogbg_gelu/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/ogbg_gelu/tuning_search_space.json rename to algorithms/target_setting_algorithms/ogbg_gelu/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/ogbg_model_size/tuning_search_space.json b/algorithms/target_setting_algorithms/ogbg_model_size/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/ogbg_model_size/tuning_search_space.json rename to algorithms/target_setting_algorithms/ogbg_model_size/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/ogbg_silu/tuning_search_space.json b/algorithms/target_setting_algorithms/ogbg_silu/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/ogbg_silu/tuning_search_space.json rename to algorithms/target_setting_algorithms/ogbg_silu/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/pytorch_adamw.py b/algorithms/target_setting_algorithms/pytorch_adamw.py similarity index 75% rename from reference_algorithms/target_setting_algorithms/pytorch_adamw.py rename to algorithms/target_setting_algorithms/pytorch_adamw.py index 14e8155d4..7220695e4 100644 --- a/reference_algorithms/target_setting_algorithms/pytorch_adamw.py +++ b/algorithms/target_setting_algorithms/pytorch_adamw.py @@ -3,14 +3,14 @@ import torch from algoperf import spec -from reference_algorithms.target_setting_algorithms import cosine_warmup -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms import cosine_warmup +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/pytorch_momentum.py b/algorithms/target_setting_algorithms/pytorch_momentum.py similarity index 80% rename from reference_algorithms/target_setting_algorithms/pytorch_momentum.py rename to algorithms/target_setting_algorithms/pytorch_momentum.py index a23b835eb..556cdd542 100644 --- a/reference_algorithms/target_setting_algorithms/pytorch_momentum.py +++ b/algorithms/target_setting_algorithms/pytorch_momentum.py @@ -4,16 +4,16 @@ from torch.optim.lr_scheduler import LambdaLR from algoperf import spec -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.jax_momentum import ( +from algorithms.target_setting_algorithms.jax_momentum import ( create_lr_schedule_fn, ) -from reference_algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/pytorch_nadamw.py b/algorithms/target_setting_algorithms/pytorch_nadamw.py similarity index 94% rename from reference_algorithms/target_setting_algorithms/pytorch_nadamw.py rename to algorithms/target_setting_algorithms/pytorch_nadamw.py index d301a233f..d6f261505 100644 --- a/reference_algorithms/target_setting_algorithms/pytorch_nadamw.py +++ b/algorithms/target_setting_algorithms/pytorch_nadamw.py @@ -7,14 +7,14 @@ from torch import Tensor from algoperf import spec -from reference_algorithms.target_setting_algorithms import cosine_warmup -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms import cosine_warmup +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/pytorch_nesterov.py b/algorithms/target_setting_algorithms/pytorch_nesterov.py similarity index 79% rename from reference_algorithms/target_setting_algorithms/pytorch_nesterov.py rename to algorithms/target_setting_algorithms/pytorch_nesterov.py index 3a6294a28..7815f1a6a 100644 --- a/reference_algorithms/target_setting_algorithms/pytorch_nesterov.py +++ b/algorithms/target_setting_algorithms/pytorch_nesterov.py @@ -4,16 +4,16 @@ from torch.optim.lr_scheduler import LambdaLR from algoperf import spec -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 +from algorithms.target_setting_algorithms.get_batch_size import ( # noqa: F401 get_batch_size, ) -from reference_algorithms.target_setting_algorithms.jax_momentum import ( +from algorithms.target_setting_algorithms.jax_momentum import ( create_lr_schedule_fn, ) -from reference_algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 update_params, ) diff --git a/reference_algorithms/target_setting_algorithms/pytorch_submission_base.py b/algorithms/target_setting_algorithms/pytorch_submission_base.py similarity index 100% rename from reference_algorithms/target_setting_algorithms/pytorch_submission_base.py rename to algorithms/target_setting_algorithms/pytorch_submission_base.py diff --git a/reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json b/algorithms/target_setting_algorithms/wmt/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json rename to algorithms/target_setting_algorithms/wmt/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/wmt_attention_temp/tuning_search_space.json b/algorithms/target_setting_algorithms/wmt_attention_temp/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/wmt_attention_temp/tuning_search_space.json rename to algorithms/target_setting_algorithms/wmt_attention_temp/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/wmt_glu_tanh/tuning_search_space.json b/algorithms/target_setting_algorithms/wmt_glu_tanh/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/wmt_glu_tanh/tuning_search_space.json rename to algorithms/target_setting_algorithms/wmt_glu_tanh/tuning_search_space.json diff --git a/reference_algorithms/target_setting_algorithms/wmt_post_ln/tuning_search_space.json b/algorithms/target_setting_algorithms/wmt_post_ln/tuning_search_space.json similarity index 100% rename from reference_algorithms/target_setting_algorithms/wmt_post_ln/tuning_search_space.json rename to algorithms/target_setting_algorithms/wmt_post_ln/tuning_search_space.json diff --git a/algorithms/template/submission.py b/algorithms/template/submission.py new file mode 100644 index 000000000..3fcb87365 --- /dev/null +++ b/algorithms/template/submission.py @@ -0,0 +1,171 @@ +"""Template submission module. + +See https://github.com/mlcommons/algorithmic-efficiency/blob/main/DOCUMENTATION.md#allowed-submissions +and https://github.com/mlcommons/algorithmic-efficiency/blob/main/DOCUMENTATION.md#disallowed-submissions +for guidelines. +""" + +from typing import Any, Dict, Iterator, List, Optional, Tuple + +from algoperf import spec + + +def get_batch_size(workload_name: str) -> int: + """Gets batch size for workload. + Note that these batch sizes only apply during training and not during evals. + + Args: + workload_name (str): Valid workload_name values are: "wmt", "ogbg", + "criteo1tb", "fastmri", "imagenet_resnet", "imagenet_vit", + "librispeech_deepspeech", "librispeech_conformer" or any of the + variants. + + Returns: + int: batch_size + + Raises: + ValueError: If workload_name is not handled. + """ + pass + + +def init_optimizer_state( + workload: spec.Workload, + model_params: spec.ParameterContainer, + model_state: spec.ModelAuxiliaryState, + hyperparameters: spec.Hyperparameters, + rng: spec.RandomState, +) -> spec.OptimizerState: + """Initializes optimizer state, e.g. for EMAs or learning rate schedules. + + Args: + workload (spec.Workload): The current workload. + model_params (spec.ParameterContainer): The current model parameters. + model_state (spec.ModelAuxiliaryState): Holds auxiliary state of the model, + such as current batch norm statistics. + hyperparameters (spec.Hyperparameters): The hyperparameters for the + algorithm. + rng (spec.RandomState): The random state. + + Returns: + spec.OptimizerState: The initialized optimizer state. + """ + pass + + +def update_params( + workload: spec.Workload, + current_param_container: spec.ParameterContainer, + current_params_types: spec.ParameterTypeTree, + model_state: spec.ModelAuxiliaryState, + hyperparameters: spec.Hyperparameters, + batch: Dict[str, spec.Tensor], + loss_type: spec.LossType, + optimizer_state: spec.OptimizerState, + eval_results: List[Tuple[int, float]], + global_step: int, + rng: spec.RandomState, + train_state: Optional[Dict[str, Any]] = None, +) -> spec.UpdateReturn: + """Updates the model parameters, e.g., a step of the training algorithm. + + Args: + workload (spec.Workload): The current workload. + current_param_container (spec.ParameterContainer): The current model + parameters. + current_params_types (spec.ParameterTypeTree): The types of the current + model parameters, e.g. weights, biases, conv, batch norm, etc. + model_state (spec.ModelAuxiliaryState): Holds auxiliary state of the + model, such as current batch norm statistics. + hyperparameters (spec.Hyperparameters): The hyperparameters for the + algorithm. + batch (Dict[str, spec.Tensor]): The current batch of data. + loss_type (spec.LossType): The type of loss function. + optimizer_state (spec.OptimizerState): The current optimizer state. + eval_results (List[Tuple[int, float]]): The evaluation results from the + previous step. + global_step (int): The current global step. + rng (spec.RandomState): The random state. + train_state (Optional[Dict[str, Any]], optional): The current training + state, e.g., accumulated submission time. + + Returns: + spec.UpdateReturn: Tuple[OptimizerState, ParameterContainer, + ModelAuxiliaryState], containing the new optimizer state, the new + parameters, and the new model state. + """ + pass + + +def prepare_for_eval( + workload: spec.Workload, + current_param_container: spec.ParameterContainer, + current_params_types: spec.ParameterTypeTree, + model_state: spec.ModelAuxiliaryState, + hyperparameters: spec.Hyperparameters, + loss_type: spec.LossType, + optimizer_state: spec.OptimizerState, + eval_results: List[Tuple[int, float]], + global_step: int, + rng: spec.RandomState, +) -> spec.UpdateReturn: + """Prepares for evaluation, e.g., switching to evaluation parameters. + Arguments are the same as `update_params`, with the only exception of batch. + + Args: + workload (spec.Workload): The current workload. + current_param_container (spec.ParameterContainer): The current model + parameters. + current_params_types (spec.ParameterTypeTree): The types of the current + model parameters, e.g. weights, biases, conv, batch norm, etc. + model_state (spec.ModelAuxiliaryState): Holds auxiliary state of the + model, such as current batch norm statistics. + hyperparameters (spec.Hyperparameters): The hyperparameters for the + algorithm. + loss_type (spec.LossType): The type of loss function. + optimizer_state (spec.OptimizerState): The current optimizer state. + eval_results (List[Tuple[int, float]]): The evaluation results from the + previous step. + global_step (int): The current global step. + rng (spec.RandomState): The random state. + + Returns: + spec.UpdateReturn: Tuple[OptimizerState, ParameterContainer, + ModelAuxiliaryState], containing the new optimizer state, the new + parameters, and the new model state. + """ + pass + + +def data_selection( + workload: spec.Workload, + input_queue: Iterator[Dict[str, spec.Tensor]], + optimizer_state: spec.OptimizerState, + current_param_container: spec.ParameterContainer, + model_state: spec.ModelAuxiliaryState, + hyperparameters: spec.Hyperparameters, + global_step: int, + rng: spec.RandomState, +) -> Dict[str, spec.Tensor]: + """Selects data from the infinitely repeating, pre-shuffled input queue. + Each element of the queue is a batch of training examples and labels. + Tip: If you would just like the next batch from the input queue return + `next(input_queue)`. + + Args: + workload (spec.Workload): The current workload. + input_queue (Iterator[Dict[str, spec.Tensor]]): The input queue. + optimizer_state (spec.OptimizerState): The current optimizer state. + current_param_container (spec.ParameterContainer): The current model + model_state (spec.ModelAuxiliaryState): Holds auxiliary state of the + model, such as current batch norm statistics. + hyperparameters (spec.Hyperparameters): The hyperparameters for the + algorithm. + global_step (int): The current global step. + rng (spec.RandomState): The random state. + + Returns: + Dict[str, spec.Tensor]: A dictionary of tensors, where the keys are the + names of the input features and the values are the corresponding tensors. + """ + pass diff --git a/docs/.old/CALL_FOR_SUBMISSIONS.md b/docs/.old/CALL_FOR_SUBMISSIONS.md deleted file mode 100644 index c5fff46d7..000000000 --- a/docs/.old/CALL_FOR_SUBMISSIONS.md +++ /dev/null @@ -1,30 +0,0 @@ -# MLCommons™ AlgoPerf: Call for Submissions - -## Announcing the AlgoPerf: Training Algorithms Benchmark Competition - -Neural networks must be trained to be useful. However, training is a resource-intensive task, often demanding extensive compute and energy resources. -To promote faster training algorithms, the [MLCommons® Algorithms Working Group](https://mlcommons.org/en/groups/research-algorithms/) is delighted to present the **AlgoPerf: Training Algorithms** benchmark. This benchmark competition is designed to measure neural network training speedups due to *algorithmic improvements*. We welcome submissions that implement both novel and existing training algorithms, including, but not limited to: - -- Optimizer update rules -- Hyperparameter tuning protocols, search spaces, or schedules -- Data sampling strategies - -Submissions can compete under two hyperparameter tuning rulesets (with separate prizes and awards): an external tuning ruleset meant to simulate tuning with a fixed amount of parallel resources, or a self-tuning ruleset for hyperparameter-free algorithms. - -## Dates - -- Call for submissions: November 28th, 2023 -- **Registration deadline to express non-binding intent to submit: February 28th, 2024**.\ -Please fill out the (mandatory but non-binding) [**registration form**](https://forms.gle/K7ty8MaYdi2AxJ4N8). -- **Submission deadline: April 04th, 2024** *(moved by a week from the initial March 28th, 2024)* -- [Announcement of all results](https://mlcommons.org/2024/08/mlc-algoperf-benchmark-competition/): August 1st, 2024 - -For a detailed and up-to-date timeline see the [Competition Rules](/COMPETITION_RULES.md). - -## Participation - -For details on how to participate in the competition, please refer to our [Competition Rules](/COMPETITION_RULES.md). To learn more about the benchmark, see our [technical documentation](/DOCUMENTATION.md). The benchmark is further motivated, explained, and justified in the accompanying [paper](https://arxiv.org/abs/2306.07179). We require all submissions to be provided under the open-source [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0). - -## Prize Money & Funding - -MLCommons has provided a total of $50,000 in prize money for eligible winning submissions. We would also like to express our gratitude to Google for their generous support in providing computational resources to score the top submissions, and resources to help score some promising submissions from submitters with more limited resources. diff --git a/docs/.old/COMPETITION_RULES.md b/docs/.old/COMPETITION_RULES.md deleted file mode 100644 index 49ad9365f..000000000 --- a/docs/.old/COMPETITION_RULES.md +++ /dev/null @@ -1,128 +0,0 @@ -# MLCommons™ AlgoPerf: Competition Rules - -**Version:** 0.0.1 *(Last updated November 14, 2023)* - -## Table of Contents - -- [Goal](#goal) -- [Sponsor](#sponsor) -- [Eligibility](#eligibility) -- [Competition Period](#competition-period) -- [Agreement to Official Rules](#agreement-to-official-rules) -- [How to Enter](#how-to-enter) -- [Submission Conditions](#submission-conditions) -- [Software Dependencies](#software-dependencies) -- [Scoring](#scoring) -- [Submissions](#submissions) -- [Optional](#optional) -- [Physical Review](#physical-review) -- [Notification](#notification) -- [Prizes](#prizes) -- [Prize Conditions](#prize-conditions) -- [Jurisdiction](#jurisdiction) -- [Cancellation and Modification](#cancellation-and-modification) -- [Publicity](#publicity) -- [Privacy](#privacy) -- [Official Rules and Winners List](#official-rules-and-winners-list) - -## Goal - -To discover new training algorithms that can train general neural networks faster. Sponsor will use an objective measuring program to allocate a score to each entry ("Submission") and determine two (2) winners (one in each ruleset), each of which will be eligible to win a prize. - -## Sponsor - -This Competition ("Competition") is sponsored by MLCommons (649 Mission Street, 5th Floor San Francisco, CA 94105, USA). - -## Eligibility - -The Competition is open to English-speaking individuals and teams (made of individuals), who are the age of majority as of the Competition start date, have internet access, a GitHub account in good standing, and can legally participate in this Competition ("Teams"). A Team may have unlimited participants, but all names must be entered. MLCommons Chairs and Sponsor's associated institutions are not eligible for prizes, but may participate. No natural person can be on multiple teams. This Competition is void wherever such competitions are prohibited. This Competition is subject to all applicable laws, including national, state, and local laws. - -## Competition Period - -The Competition begins at 12:01am (ET) on November 28, 2023 and ends at 11:59pm (ET) on May 28, 2024, all according to Sponsor's time clock, which decisions are final (the "Competition Period"). There are several deadlines contained within the Competition Period: - -- **Intention to Submit.** You must register your Intention to Submit no later than 11:59pm ET on February 28, 2024. -- **Submission Period.** You must complete your Submission and enter it after the Intention to Submit deadline, but no later than 11:59pm ET on April 04, 2024. - -## Agreement to Official Rules - -By participating, Teams agree to be fully unconditionally bound by these Rules, and you represent and warrant that you meet the eligibility requirements set forth herein. In addition, you agree to accept the decisions of Sponsor, as final and binding, and waive any right to claim ambiguity in the Competition or these Rules. - -## How to Enter - -There are four (4) steps to a successful submission ("Submission"). - -1. **Register Intent to Submit.** Registration of intent does not obligate you to enter a Submission, but you must register prior to entering your Submission. Click for the [Intent Form](https://forms.gle/K7ty8MaYdi2AxJ4N8). This is your "Team," even if you are a single person. Please note that natural persons may not be on multiple teams, but each Team may enter multiple Submissions. -2. **Develop your Submission.** Develop your Submission according to the guidelines set forth in these rules, along with the links to various necessary information. Please note that all Submissions must be entered subject to the Apache 2.0 license. In order to develop your Submission, you must: - - *Fork the Benchmark Codebase.* Begin by creating a (public or private) GitHub repository for your contest submission. Once you submitted, this repository must be a clone of the frozen main branch of the benchmark codebase. Ensure that all elements of the original codebase remain unaltered, with the exception of the `/submission` directory. - - *Preserve the Apache 2 License.* You must maintain the same Apache 2 License for your repository as the benchmark codebase. This means you may not change the licensing terms. Submissions that change the terms or otherwise fail to maintain the license, will be deemed ineligible submissions. - - *Define Software Dependencies.* If your Submission will have any software dependencies, you must create a `requirements.txt` file in the `/submission` directory. This file must clearly list all software dependencies your Submission requires in order to be a valid Submission. File must be "pip readable" (the dependencies listed can be installed via the `pip install -r requirements.txt` command). You may not modify the package versions of the software dependencies used by the benchmarking codebase, including using a different version of libraries such as PyTorch or JAX from those specified in the benchmark. -3. **Submit Your Entry & Complete Your Submission Forms.** During the Submission Period, once your Submission is complete, submit it using the [Submission Form](https://forms.gle/yXQqwJ6Nm6sszPw49). The submission form must contain the URL of your submission's GitHub repository and the following agreements: - - A signed [Contributor License Agreement (CLA) "Corporate CLA"](https://mlcommons.org/en/policies/) of MLCommons. - - *Either* a membership in MLCommons *or* a signed [non-member test agreement](https://mlcommons.org/en/policies/). - - A signed trademark license agreement, either the member or the non-member version, as appropriate. These license agreements are available upon request to [support@mlcommons.org](mailto:support@mlcommons.org). - - The form is sent to the working group chairs, who will process your Submission. Failure to complete the proper Submission Forms will results in disqualification of your Submission. At the close of the Submission Period, your GitHub repository must be public. - -## Submission Conditions - -All Submissions must meet the requirements of the terms contained in these rules, including reliance on new algorithmic or mathematical ideas and concepts, and must not use software engineering approaches in order to increase primitive operations in PyTorch, JAX, their dependencies, the operating systems, or the hardware. By entering, all Team members warrant that their Submission does not infringe any third party's rights, and that Team members have obtained all necessary permissions from all relevant third parties to submit the Submission. If, in the sole discretion of Sponsor, any Submission constitutes copyright or other intellectual property infringement, the Submission will be disqualified. Team must hold all rights through license or ownership to the entire Submission. Team members agree to indemnify Sponsor against any and all claims of infringement from any third party for any use by Sponsor of a Submission. Team members may not be: 1) represented under contract that would limit or impair Sponsor's ability to use the Submission; or 2) are under any other contractual relationship, including but not limited to guild and/or union memberships, that may prohibit them from participating fully in this Competition, or from allowing Sponsor to use royalty-free, the Submission worldwide in all media in perpetuity. - -No Submission may depict any offensive or obscene subject matter as determined in Sponsor's sole discretion. No Submission shall portray Sponsor in a negative light. The Submission will be deemed to be owned equally by all team members, regardless of any agreement between the team members, which will not be honored by Sponsor. A Submission may be disqualified by Sponsor, in its sole discretion, if they violate the spirit and goodwill of the rules, including without limitation, if Sponsor determines a Submission is a slavish copy or derivative work of a third party that was previously developed. Submissions will be disqualified if they circumvent any rules, or protocols, including circumventing the tuning rules by looking up the result of an offline computation performed ahead of time; computing any form of pairwise metrics between the fixed and held-out workloads. Submission may use public APIs in JAX and PyTorch from within the submission function APIs, but may not use APIs to optimize the internals of primitive operations and/or standard dependencies to benefit any Submission. - -## Software Dependencies - -Submissions must use specific versions of PyTorch and JAX, provided by Sponsor. Additional dependencies may be added, provided Teams include a description of the additions and their function. Submissions can include dependencies that support new algorithmic and mathematical ideas provided they do not circumvent the intention of the benchmark in any way that changes measurement of the training speeds. - -## Scoring - -All otherwise qualified Submissions shall be scored. Submissions will be scored based on their required training time to reach the target performance on the validation set of each workload, using measuring techniques designed to give all Submissions equal parity. In the event that no Submission in a ruleset receives a score exceeding that of both [prize qualification baselines](./prize_qualification_baselines/README.md), no prizes will be awarded for this ruleset. The Teams with the highest scores will be determined to be winners ("Selected Teams"). In the event of a tie the prize money will be split equally between the winners. - -## Submissions - -Teams may enter as many Submissions as they like during the Submission Period and all otherwise qualified Submissions will be scored. - -## Optional - -Team members may join the Algorithm mailing group, located [here](https://groups.google.com/u/4/a/mlcommons.org/g/algorithms). This mailing group provides information to Teams regarding the status of the Competition. - -## Physical Review - -All Submission are subject to human review and testing to determine whether, in Sponsor's sole and exclusive discretion, any Submission fails to comply with the spirit of the Competition, and is thus disqualified. Both physical review team and other judges shall be qualified to judge the Competition. - -## Notification - -On or about July 15, 2024, the Selected Team with the best scores as determined by Sponsor will be notified that they are potential winners of the Competition. The Selected Team will be notified by either phone or email at the sole discretion of Sponsor or Sponsor's representative. Selected Team will be required to respond (as directed) to a phone and/or e-mail notification within 72 hours of attempted notification. The failure to respond timely to the notification may result in forfeiture of the prize; and, in such case, Sponsor may choose the next highest scoring Submission from among the remaining eligible Submissions. Selected Team members will each be required to sign and return a Declaration (or affidavit, at Sponsor's option) of Eligibility and Liability/Publicity Release ("Declaration") and any other documents Sponsor or Sponsor's representative may require within 72 hours of receipt of the Declaration. Failure to timely return a signed Declaration (or failure of a Team member to return it), or any other required documents or the return of any prize notification as undeliverable will result in Prize forfeiture. National and state income taxes may apply and are the sole responsibility of the winner. All expenses not specifically stated as being included are excluded, and are the responsibility of the Selected Teams. No assignment, transfer or substitution of Prize is permitted, however, Sponsor reserves the right to substitute a prize for one of comparable or greater value should Prize become impracticable to award or unavailable for any reason. - -## Prizes - -There will be two prizes awarded, one per each ruleset. Prizes will be awarded in US Dollars. Prize will be awarded in cash, or as a gift card, at Sponsor's option. In the event the prize is a gift card, Team will be required to accept the terms and conditions of gift card. Prizes will be divided evenly among enumerated Team members listed as of the date of the Submission. In the event Sponsor is unable to award the prize, as outlined herein, for any reason, Sponsor may substitute a prize of equal or greater value. - -- "Best Performance '*external-tuning*'": US $25,000 -- "Best Performance '*self- tuning*'": US $25,000 - -## Prize Conditions - -For all prizes, all national, state, province, and local taxes and other expenses in connection with the prize not expressly described herein as being awarded are the sole responsibility of the Selected Contestant. Selected Teams are solely responsible for any other unspecified expenses related to prize. Selected Teams cannot assign their prize to another person. No substitution of prize, provided however that Sponsor reserves the right to substitute a prize with another prize of equal or greater value. In the event of noncompliance with the foregoing requirements or if prize notification is returned as undeliverable, prize will be forfeited and, at Sponsor's discretion, an alternate Selected Teams with the next highest score will be chosen. - -Competition is subject to these Official Rules. By participating, Teams agree: (i) to be bound by these complete Official Rules and the decisions of Sponsor which shall be final and binding; and (ii) to waive any right to claim ambiguity in the Competition or these Official Rules, except where prohibited by law. By participating in Competition or by accepting a prize, Selected Team agrees to release Sponsor, including its parent, subsidiary and affiliated entities together with the respective directors, employees, officers, licensees, licensors and agents, and respective advertising and promotion entities and any person or entity associated with the production, judging, or administration of the Competition (collectively, the "Releasees") from any and all liability, loss or damage arising from or in connection with awarding, receipt and/or use or misuse of prize or participation in any prize-related activities. Releases shall not be liable for: (i) telephone system, telephone or computer hardware, software or other technical or computer malfunctions, lost connections, disconnections, delays or transmission errors; (ii) data corruption, theft, destruction, unauthorized access to or alteration of entry or other materials; (iii) any injuries, losses or damages of any kind, including death, caused by the use of the prize money, or resulting from acceptance, possession or use of a prize, or from participation in the Competition; or (iv) any printing, typographical, administrative or technological errors in any materials associated with the Competition. Sponsor disclaims any liability for damage to any computer system resulting from participating in, or accessing or downloading information, including licenses and other information germane to the running of the Competition or otherwise in connection with this Competition. Sponsor reserves the right to cancel or suspend the Competition, in its sole discretion, should it receive fewer than two (2) prize money-eligible Submissions per ruleset, or receive no Submissions that have a judged score above a threshold set by the Sponsor, or due to circumstances beyond its control, including natural disasters, pandemic, computer virus, excessive cheating, or any other event that would undermine the fair play of the Competition. Submissions will not be returned and may be destroyed. - -## Jurisdiction - -The internal laws of the State of California in the United States of America will govern disputes regarding these Official Rules and/or this Contest. All cases and claims pertaining to this Contest must be brought in a court of competent jurisdiction in the City of San Francisco. - -## Cancellation and Modification - -Sponsor reserves the right, in its sole discretion, to cancel, modify or suspend the Competition should a virus, bug, computer problem, unauthorized intervention or other causes beyond Sponsor's control, corrupt the administration, security or proper play of the Competition. Sponsor reserves the right to cancel the competition should it receive fewer than two (2) prize money-eligible submissions per ruleset, or which are not above a threshold score as noted in these rules. Sponsor may prohibit an entrant Team (or a single person) from participating in the Competition or winning prize if, in its sole discretion, it determines such entrant is attempting to undermine the legitimate operation of the Competition in any way by cheating, hacking, deception, or any other unfair practices, including intention to annoy, abuse, threaten or harass any other competitors or Sponsor representatives. Any attempts to circumvent safeguards and benchmarks will result in disqualification, including the relevant IP address becoming ineligible for the entire Competition. Caution: any attempt to deliberately damage or undermine the legitimate operation of the Competition may be in violation of criminal and civil laws and will result in disqualification from participation in the contest. Should such an attempt be made, Sponsor reserves the right to seek remedies and damages (including attorney fees) to the fullest extent of the law, including criminal prosecution. - -## Publicity - -Except where prohibited, all entrants agree that Sponsor, its shareholders, agents and representatives, affiliates, subsidiaries, advertising, promotion and fulfillment agencies, and legal advisors are not responsible or liable for, and shall be released and held harmless from any and all losses, damages, rights, claims and actions of any kind in connection with or resulting from participation in the Contest, or acceptance of the prize, including without limitation, claims based on publicity rights, defamation, or invasion of privacy. Except where prohibited by law, Sponsor reserves the right to use the Submissions to the Competition, in whole or in part, for publicity purposes prior to, during, or after the Competition, in any media, and to use the name, likeness, hometown name, of any Contestant, including all or part of their Submission throughout the world, in perpetuity, without any compensation or prior review unless specifically prohibited by law. Except as outlined herein for winners, Teams and their members will not be paid for their Submissions or for granting Sponsor any of these rights. Should any Selected Team be unwilling or otherwise unable to provide permissions and or releases or otherwise cannot accept or receive the prize for any reason, the Selected Team with the next highest score will be chosen from the remaining entries until one who is able to meet all requirements can be selected - -## Privacy - -All personal information collected by Sponsor will be used for administration of the Competition. In addition, Team members may receive email correspondence from, or on behalf of Sponsor, via electronic communication relating to the Competition. All personal information will be held on servers located in the United States. Sponsor will use reasonable commercial efforts to comply with Federal CAN-SPAM guidelines and other privacy guidelines, and US residents may receive commercial communications, which they may subsequently opt-out of receiving further advertising emails by following the opt-out instructions contained in any email communications received. - -## Official Rules and Winners List - -For a copy of these Official Rules or of the winner(s) of this Competition, send your request via email to [algorithms-chairs@mlcommons.org](mailto:algorithms-chairs@mlcommons.org). The Request and the request must be received within 90 days of the Competition end date. Please allow a reasonable time for a response. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 685926506..419553cd6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,13 +1,72 @@ -# Change Log +# Changelog -## algoperf-benchmark-0.1.5 (2024-03-26) +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (additionally tagging whether a change affects the [Code, Docs, Rules, Leaderboard]), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +**Versioning Policy:** +AlgoPerf uses a unified versioning scheme: codebase, rules, and leaderboard all share the same `Major.Minor` version. All results produced under the same `Major.Minor` version are comparable. `Patch` versions can be incremented independently for each component to reflect smaller, non-breaking changes to allow some flexibility: + +- _Leaderboard_: New submissions or minor fixes to the leaderboard could increment its `Patch` version (e.g., `0.6.0` -> `0.6.1`) as shown in the leaderboard repo. +- _Codebase_: API improvements, bug fixes, or small non-breaking changes in the benchmark code could increment its `Patch` version as reflected in the `algoperf` package version. +- _Documentation/Rules_: Clarifications, typo fixes, or minor updates to the rules/documentation could increment its `Patch` version as shown in the documentation file. + +## [0.6.0] - 2025-06-24 + +Improved and streamlined version of the benchmark which includes important bug fixes, API improvements and benchmark protocol changes following the lessons learned from the first competition. + +### Added + +- [Code, Rules] Updated API to allow for `prepare_for_eval` function ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/806)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/789)). +- [Docs] Document default dropout values for each workload ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/806)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/786)). +- [Docs] Unified versioning policy section ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/876)). +- [Code] Add the ability to change dropout values during training ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/875)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/753)). + +### Changed/Removed + +- [Code, Docs] Rename package to `algoperf` ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/833)). +- [Code, Docs] Switch to `ruff` for linting and formatting([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/874)). +- [Code, Rules] Pass `train_state` to `update_params` function ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/790)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/785)). +- [Code, Rules] Reduced number of studies from 5 to 3 ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/876)). See also Section 5.1 in our [results paper](https://arxiv.org/abs/2502.15015). +- [Code, Rules] Remove held-out workloads from the benchmark ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/876)). See also Section 5.1 in our [results paper](https://arxiv.org/abs/2502.15015). +- [Code] Remove sacrebleu dependency ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/828)). +- [Code] Switch to `pyproject.toml` for package management ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/830)). +- [Code] Update Python version to 3.11 and dependencies accordingly ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/811)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/805)). +- [Rules] Modify the runtime budgets and step hints for each workload ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/838)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/836)). See also Section 5.1 in our [results paper](https://arxiv.org/abs/2502.15015). +- [Code] Automatically determine the package version via the latest GitHub tag ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/831)). +- [Code, Docs] Move all algorithms into a dedicated `algorithms` directory ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/876)). +- [Code] Migrate from `pmap` to `jit` in JAX for better performance and scalability ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/848)). + +### Fixed + +- [Code] Batch norm bug ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/783)/[PR](https://github.com/mlcommons/algorithmic-efficiency/pull/798)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/767)). +- [Code] Fix bug of potentially giving a free evaluation to a submission that goes out of `max_runtime` ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/789)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/719#issuecomment-2328797610)). +- [Code] Fix that models in the self-tuning ruleset will always be initialized with default dropout ([PR](https://github.com/mlcommons/algorithmic-efficiency/pull/873)/[PR](https://github.com/mlcommons/algorithmic-efficiency/pull/875)/[Issue](https://github.com/mlcommons/algorithmic-efficiency/issues/753)). + +## [0.5.0] - 2024-03-26 + +The version of the benchmark used for the first competition. + +**Summary:** - Finalized variant workload targets. - Fix in random_utils helper function. - For conformer PyTorch Dropout layers set `inplace=True`. - Clear CUDA cache at begining of each trial for PyTorch. -## algoperf-benchmark-0.1.4 (2024-03-26) +**What's changed:** + +- update speech variants target setting points by @priyakasimbeg in #727 +- set num_workers for librispeech back to 4 by @priyakasimbeg in #736 +- [fix] random_utils.py to `_signed_to_unsigned` by @tfaod in #739 +- Fix path in helper config for running experiments in bulk. by @priyakasimbeg in #740 +- Finalize variants targets by @priyakasimbeg in #738 +- Aiming to Fix Conformer OOM by @pomonam in #710 +- Lint fixes by @priyakasimbeg in #742 +- Add warning for PyTorch data loader num_workers flag. by @priyakasimbeg in #726 + +## [0.0.4] - 2024-03-26 Upgrade CUDA version to CUDA 12.1: @@ -28,7 +87,7 @@ Added instructions for submission. Changed default number of workers for PyTorch data loaders to 0. Running with >0 may lead to incorrect eval results see . -## algoperf-benchmark-0.1.2 (2024-03-04) +## [0.0.3] - 2024-03-04 Workload variant additions and fixes: @@ -45,12 +104,12 @@ Datasetup fixes. Fix tests that check training differences in PyTorch and JAX on GPU. -## algoperf-benchmark-0.1.1 (2024-01-19) +## [0.0.2] - 2024-01-19 Bug fixes to FastMRI metric calculation and targets. Added workload variants and targets for ogbg, fastmri, librispeech_conformer, imagenet_resnet, imagenet_vit, criteo1tb to be used as held-out workloads. -## algoperf-benchmark-0.1.0 (2023-11-28) +## [0.0.1] - 2023-11-28 First release of the AlgoPerf: Training algorithms benchmarking code. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index e9918e14c..f635b977f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -23,12 +23,13 @@ - [Unit and Integration Tests](#unit-and-integration-tests) - [Regression Tests](#regression-tests) - [Versioning](#versioning) + - [Release Process](#release-process) ## Contributing to MLCommons We invite everyone to look through our technical documentation and codebase and submit issues and pull requests, e.g. for changes, clarifications, or any bugs you might encounter. If you are interested in contributing to the work of the working group and influence the benchmark's design decisions, please [join the weekly meetings](https://mlcommons.org/en/groups/research-algorithms/) and consider becoming a member of the working group. -The best way to contribute to the MLCommons is to get involved with one of our many project communities. You find more information about getting involved with MLCommons [here](https://mlcommons.org/en/get-involved/#getting-started). +The best way to contribute to the MLCommons is to get involved with one of our many project communities. You can find more information about getting involved with MLCommons on our [getting started page](https://mlcommons.org/en/get-involved/#getting-started). Generally we encourage people to become a MLCommons member if they wish to contribute to MLCommons projects, but outside pull requests are very welcome too. @@ -46,12 +47,12 @@ We recommmend to use the Deep Learning on Linux image. Further instructions are ### Installing GPU Drivers -You can use the `scripts/cloud-startup.sh` as a startup script for the VM. This will automate the installation of the NVIDIA GPU Drivers and NVIDIA Docker toolkit. +You can use the `docker/scripts/cloud-startup.sh` as a startup script for the VM. This will automate the installation of the NVIDIA GPU Drivers and NVIDIA Docker toolkit. ### Authentication for Google Cloud Container Registry To access the Google Cloud Container Registry, you will have to authenticate to the repository whenever you use Docker. -Use the gcloud credential helper as documented [here](https://cloud.google.com/artifact-registry/docs/docker/pushing-and-pulling#cred-helper). +Use the gcloud credential helper as documented in the [Google Cloud documentation](https://cloud.google.com/artifact-registry/docs/docker/pushing-and-pulling#cred-helper). ## Installation @@ -246,7 +247,7 @@ To automatically fix linting errors you can run `ruff check --fix`, with the add ### Unit and Integration Tests We run unit tests and integration tests as part of the of github actions as well. -You can also use `python tests/reference_algorithm_tests.py` to run a single model update and two model evals for each workload using the reference algorithm in `reference_algorithms/target_setting_algorithms/`. +You can also use `python tests/reference_algorithm_tests.py` to run a single model update and two model evals for each workload using the reference algorithm in `algorithms/target_setting_algorithms/`. ### Regression Tests @@ -268,12 +269,35 @@ To run a regression test: ### Versioning +AlgoPerf uses a unified versioning scheme: codebase, rules, and leaderboard all share the same `Major.Minor` version. `Patch` versions may differ for minor updates to each component. All results produced under the same `Major.Minor` version should be comparable. See the [README](../README.md#releases--roadmap) and [Changelog](CHANGELOG.md) for details. + The package version is automatically determined by the `setuptools_scm` package based on the last git tag. It follows the structure `major.minor.patch` + `devN` where `N` is the number of commits since the last tag. It automatically increments the patch version (i.e. it guesses the next version) if there are commits after the last tag. Additionally, if there are uncommitted changes, the version will include a suffix separated by a `+` character and includes the last commit hash plus the date on dirt workdir (see [setuptools_scm's documentation](https://setuptools-scm.readthedocs.io/en/latest/extending/#setuptools_scmlocal_scheme) with the default version and local scheme). You can check what version `setuptools_scm` is creating by running `python -m setuptools_scm`. -To create a new version, create a new release (and tag) in the GitHub UI. -The package version is automatically updated to the new version. -Once the package is installed, the version can be accessed as the package attribute `algoperf.__version__`, i.e. via `python -c "import algoperf; print(algoperf.__version__)"`. +#### Release Process + +The suggested workflow: + +- **Development:** + + - All changes will be on the `dev` (or `dev-0.X` or similar) branch. Only merge to `main` once we release. + - For internal milestones, we could use pre-release labels like `-alpha.N`, `-beta.N` or `-rc.N`. + - Iterative changes here, do not increment the version, since on this branch we are working _towards_ the next release. + - All changes should be documented in the `CHANGELOG.md` for the upcoming version release. This includes changes in the code and the rules. + - Do **not** manually edit version numbers in the codebase or `pyproject.toml`. + +- **Changes:** All changes that affect the results of the benchmark should result in increases to either the `Major` or `Minor` version. We could reserve increases to the `Major` version for larger changes like adding new workloads. Changes that do not affect the results of the benchmark should result in increases to the `Patch` version and could include the following: + + - _Codebase:_ Implement bug fixes, improvements, or new features. The git tag version automatically updates the `algoperf.__version__` of the package. + - _Documentation/Rules:_ Updates like clarifications, typo fixes, or new content. Update the version in `docs/DOCUMENTATION.md` with the new version. + - _Leaderboard:_ For example, adding a new submission, correcting typos, or adding details could result in updating the version as documented in the `submissions_algorithms` repo. + +- **Release new version:** + - Check that `CHANGELOG.md` is up-to-date and complete. + - Update the version in `docs/DOCUMENTATION.md` with the new version. + - Update the release plan in the [README](../README.md#releases--roadmap) with the new version. + - Merge `dev` or `dev-0.X` into `main`. + - Tag release with new version in the GitHub UI. The package version is automatically updated to the new version. Once the package is installed, the version can be accessed as the package attribute `algoperf.__version__`, i.e. via `python -c "import algoperf; print(algoperf.__version__)"`. diff --git a/docs/DOCUMENTATION.md b/docs/DOCUMENTATION.md index 758361968..f7ac5e659 100644 --- a/docs/DOCUMENTATION.md +++ b/docs/DOCUMENTATION.md @@ -1,169 +1,111 @@ -# MLCommons™ AlgoPerf: Technical Documentation & FAQs +# MLCommons™ AlgoPerf: Documentation, Benchmarking Process & FAQs -**Version:** 0.0.22 *(Last updated February 10, 2025)* +**Version:** 0.6.0 _(Last updated August 27, 2025)_ -> **TL;DR** New training algorithms and models can make neural net training faster. -> We need a rigorous training time benchmark that measures time to result given a fixed hardware configuration and stimulates algorithmic progress. We propose a *Training Algorithm Track* and a *Model Track* in order to help disentangle optimizer improvements and model architecture improvements. This two-track structure lets us enforce a requirement that new optimizers work well on multiple models and that new models aren't highly specific to particular training hacks. The following is the technical documentation for the Training Algorithm Track. +> [!IMPORTANT] +> +> **TL;DR:** The MLCommons™ **AlgoPerf: Training Algorithms benchmark is designed to find training algorithms that can train neural networks faster** by rigorously measuring how quickly they reach a specific performance target across a diverse set of deep learning workloads. +> This document provides the technical documentation, benchmarking process, and FAQs for the AlgoPerf benchmark. ## Table of Contents - [Introduction](#introduction) -- [Technical Documentation of the Training Algorithm Track](#technical-documentation-of-the-training-algorithm-track) + - [Motivation](#motivation) + - [Overview](#overview) +- [Benchmarking Process](#benchmarking-process) - [Submissions](#submissions) - - [Specification](#specification) - - [Evaluation during training](#evaluation-during-training) - - [Valid submissions](#valid-submissions) - - [Tuning](#tuning) - - [External tuning ruleset](#external-tuning-ruleset) - - [Self-tuning ruleset](#self-tuning-ruleset) + - [Submission API](#submission-api) + - [Valid Submissions](#valid-submissions) + - [Runtime Environment and Evaluation](#runtime-environment-and-evaluation) + - [Tuning Rulesets](#tuning-rulesets) + - [External Tuning Ruleset](#external-tuning-ruleset) + - [Self-Tuning Ruleset](#self-tuning-ruleset) - [Workloads](#workloads) - - [Fixed workloads](#fixed-workloads) - - [Randomized workloads](#randomized-workloads) - - [Qualification set](#qualification-set) + - [Recommended Qualification Set](#recommended-qualification-set) - [Scoring](#scoring) - - [Benchmarking hardware](#benchmarking-hardware) - - [Defining target performance](#defining-target-performance) - - [Benchmark score using performance profiles](#benchmark-score-using-performance-profiles) - - [Version freeze](#version-freeze) + - [AlgoPerf Benchmark Score via Integrated Performance Profiles](#algoperf-benchmark-score-via-integrated-performance-profiles) + - [Benchmarking Hardware](#benchmarking-hardware) + - [Defining Target Performance and `max_runtime`](#defining-target-performance-and-max_runtime) +- [Versioning Policy](#versioning-policy) + - [Version Freeze](#version-freeze) +- [License and Legal Requirements](#license-and-legal-requirements) - [FAQs](#faqs) - - [Setup and Platform](#setup-and-platform) - - [My machine only has one GPU. How can I use this repo?](#my-machine-only-has-one-gpu-how-can-i-use-this-repo) - - [How do I run this on my SLURM cluster?](#how-do-i-run-this-on-my-slurm-cluster) - - [How can I run this on my AWS/GCP/Azure cloud project?](#how-can-i-run-this-on-my-awsgcpazure-cloud-project) + - [Setup \& Platform](#setup--platform) - [Submitting](#submitting) - - [How do I submit my algorithm to the benchmark?](#how-do-i-submit-my-algorithm-to-the-benchmark) - - [Can I submit multiple times to the benchmark competition?](#can-i-submit-multiple-times-to-the-benchmark-competition) - - [Can my submission be structured using multiple files?](#can-my-submission-be-structured-using-multiple-files) - - [Can I install custom dependencies?](#can-i-install-custom-dependencies) - - [How can I know if my code can be run on benchmarking hardware?](#how-can-i-know-if-my-code-can-be-run-on-benchmarking-hardware) - - [This benchmark seems computationally expensive. Do I have to run it myself?](#this-benchmark-seems-computationally-expensive-do-i-have-to-run-it-myself) - - [Can I submit previously published training algorithms as submissions?](#can-i-submit-previously-published-training-algorithms-as-submissions) + - [Scoring \& Hardware](#scoring--hardware) - [Disclaimers](#disclaimers) - - [Shared Data Pipelines between JAX and PyTorch](#shared-data-pipelines-between-jax-and-pytorch) + - [Shared Data Pipelines between `JAX` and `PyTorch`](#shared-data-pipelines-between-jax-and-pytorch) ## Introduction -We need a more scientifically sound methodology for evaluating training speedups due to new algorithms, including both new optimizers and new model architectures. Cutting edge machine learning (ML) models are exceeding the compute budgets of many researchers, and ML compute is becoming a larger and larger cost in industry. To reduce the compute and potentially environmental cost of ML research and practice, we need rigorous benchmarking of efficiency. Such benchmarks will guide us in selecting the best directions to evolve existing techniques and ultimately enable progress toward models that produce not only better results, but better results **at lower cost**. +### Motivation -MLCommons' mission is to build fair and useful benchmarks for measuring training and inference performance of ML hardware, software, and services. Improvements in training speed can come from better hardware, better software stacks, and better algorithms. -To date, the Closed Division of the MLPerf™ Training benchmark has been extremely successful in driving systems innovation by requiring mathematical equivalence to a reference implementation, while still allowing submissions on different hardware. Although the Open Division allows new models and training algorithms, it has several issues that make it inappropriate as a benchmark for progress in training algorithms. By allowing arbitrary hardware, it is impossible to isolate improvements due to algorithms or due to extra computation. Unrestricted hardware makes the benchmark only accessible to the most well-funded organizations, even if many academic labs and others have interesting algorithms to measure. Finally, even if we could isolate improvements due to particular algorithmic changes and make the benchmark more broadly accessible, there is still no incentive to avoid hyper-specific changes that only help the particular benchmark workload. +Neural networks are powerful models, but they need to be trained to be useful. Training cutting-edge machine learning (ML) models exceeds the compute budgets of many researchers and is a growing cost in industry. +Additionally, when training neural nets, practitioners face many critical yet often opaque decisions: What optimizer to choose? How should its learning rate be tuned? What learning rate schedule should be used? These choices can make or break training, yet the community has lacked a clear, standardized way to identify the state of the art. -In order to drive innovation in machine learning algorithms that reduce the time needed to create useful models, we propose a new set of benchmarks called **AlgoPerf** to evaluate the training time for different algorithms (models, optimizers, preprocessing, etc.) on a **fixed hardware configuration** (future iterations can adopt new hardware configurations as needed). Our proposal includes two tracks: (1) the **Training Algorithm Track** and (2) the **Model Track**. The goal of the Training Algorithm Track is to find training algorithms (optimizers, etc.) that train benchmark models to reach the goal out-of-sample error rate as fast as possible. However, to incentivize practically useful algorithms, in the Training Algorithm Track we require that a single training algorithm simultaneously performs well across all benchmark models and datasets. Similarly, the goal of the Model Track is to find models that can be trained to achieve the target solution quality (out-of-sample error) in the least amount of time on each benchmark dataset. Although submissions in the Model Track will be inherently dataset-specific, we sharply constrain what parts of the training program can be modified in the Model Track and require submitted models to be easily trainable using standard optimizers. Thus the two-track structure discourages overly specific solutions that aren't generally useful to practitioners and will hopefully produce evidence on the relative returns of speeding up training by finding new models or by developing new training algorithms. +To reduce the compute and potentially environmental cost of machine learning models, as well as provide guidance for practitioners, we need more scientifically sound methods for evaluating training speedups due to new algorithms. -In the following, we will focus on the **Training Algorithm Track** of the *AlgoPerf benchmark*. +Unlike benchmarks focused on hardware or model architecture, AlgoPerf isolates the training algorithm itself, which includes the optimizer, regularization, data selection, and hyperparameters like the learning rate schedule. By standardizing the benchmarking process, AlgoPerf offers a meaningful apples-to-apples comparison of training algorithms. -## Technical Documentation of the Training Algorithm Track +This document focuses on the **Training Algorithm Track** of the _AlgoPerf benchmark_. -The goal of the **AlgoPerf: Training Algorithm Track** is to reach the same results faster ("time to result") by using better optimizers, data ordering/weighting schemes, and weight update strategies while producing techniques that work well on a wide variety of models and datasets. We hope to encourage generally useful training algorithms that are not specific to only a small number of particular workloads. +### Overview -In general, submissions to the Training Algorithm Track will replace specific pieces of a reference implementation in order to produce a training program that reaches the same results faster on as many workloads as possible. The training program has a fixed, high-level structure and competitors are allowed to replace a particular set of functions in the program (the [**submission functions**](#submission-functions)), but must leave all other pieces ([**fixed functions**](#fixed-functions) and high-level structure) of the reference implementation unchanged. The submitted code must perform well on multiple datasets and models simultaneously (a model and dataset pair constitute a [workload](#workloads) for the purposes of this track). +The **AlgoPerf: Training Algorithms benchmark** challenges participants to submit training algorithms that accelerate the training of neural networks. The goal is to reach a pre-defined performance target in the shortest possible time ("time-to-result") across a diverse set of workloads. The benchmark is designed to identify general-purpose training algorithms, such as new optimizers, data selection methods, regularization techniques, etc., that provide practical speedups for the broader ML community. -Submissions to the Training Algorithm Track can be entered under two separate rulesets, named [external tuning ruleset](#external-tuning-ruleset) and [self-tuning ruleset](#self-tuning-ruleset), with it being possible to submit to both rulesets. The main difference is that the external tuning ruleset allows moderate, automatic, parallel tuning of the optimizer's hyperparameters on each workload, using the submitted workload-agnostic search space. This allows the training algorithm to adapt to a particular task while ensuring that it is not too difficult to tune automatically. Under the self-tuning ruleset, there is no external tuning and submissions need to adapt to a particular task autonomously within a single optimization run. Unless otherwise specified, the rules in this section apply to both rulesets (see, for example, the [Tuning](#tuning) section for the most substantial difference between the rulesets). +The benchmarking process follows these **key principles**: -The intention is that a training algorithm submission will be broadly applicable and useful without customization to the specific [workload](#workloads) (model, dataset, loss function). We want to discourage detecting the particular workload and doing something highly specific that isn't generally useful. In order to further discourage submissions that overfit to the particular [fixed benchmark workloads](#fixed-workloads), submissions will also be evaluated on [held-out workloads](#randomized-workloads) specified after the submission deadline. +- 🎯 **Fixed Target, Model & Hardware:** Submitted training algorithms must train a set of [**fixed models**](#workloads) to a pre-defined validation performance target as fast as possible. All submissions use the same model architecture and are run on the same [**standardized hardware**](#benchmarking-hardware) (currently `8x NVIDIA V100 GPUs`). This isolates the training algorithm's performance and allows a fair apples-to-apples comparison. +- ⏱️ **Time-To-Result:** Submissions are evaluated based on the total wall-clock time required to reach the target, rewarding practical and efficient algorithms. +- 🧠 **Diverse Workloads:** The benchmark includes [**8 diverse deep learning workloads**](#workloads) across domains like image classification, speech recognition, and machine translation. A submission's score is computed by aggregating its performance across all workloads, using [**performance profiles**](#algoperf-benchmark-score-via-integrated-performance-profiles), to ensure general-purpose algorithms. +- 📦 **Fully-Specified Algorithms:** Submissions must be [**complete procedures**](#submission-api) and thus hyperparameter tuning is treated as part of the algorithm. Depending on the [**ruleset**](#tuning-rulesets), submissions may use parallel tuning resources. This ensures that the benchmark measures the _total_ practical cost of a training algorithm and provides practitioners with a complete method, eliminating the guesswork of how to apply it. -### Submissions - -A valid submission is a piece of code that defines all of the submission functions and is able to train all benchmark workloads on the [benchmarking hardware](#benchmarking-hardware) (defined in the [Scoring](#scoring) section). Both the validation set and the test set performance will be checked regularly during training (see the [Evaluation during training](#evaluation-during-training) section), however, only the validation performance is relevant for scoring. Training halts when the workload-specific [target performance](#defining-target-performance) for the validation and test sets have been reached. For each workload, only the training time to reach the *validation* set target error is used as input to the [scoring process](#scoring) for the submission. Submissions using [external tuning](#external-tuning-ruleset) will be tuned independently for each workload using a single workload-agnostic search space for their specified hyperparameters. The tuning trials are selected based on the time to reach the *validation* target. Submissions under either tuning ruleset may always self-tune while on the clock. - -#### Specification - -Any function defined in the reference implementations that isn't a [submission function](#submission-functions) is a [fixed function](#fixed-functions) for the Training Algorithm Track. No submitted code is run to compute the evaluation metrics in the Training Algorithm Track. We just use the final model parameters and the fixed functions from this track at test time. - -In principle, submissions are allowed to use the available hardware systems in any data- or model-parallel manner they desire, within the constraints of the submission function APIs. However, in practice, model-parallelism may not be possible with the API. They are allowed to access any framework-specific device information necessary to exploit the hardware. - -Submissions provide a [per-workload batch size](#batch-size-getter) to use. Specification of the batch size for each workload is necessary to avoid running out of memory for different workloads. Therefore, submitters can determine this batch size in advance and specify it as part of the submission. Submitters may also provide per-workload batch sizes for all [randomized workloads](#randomized-workloads). If no such batch size is provided for a randomized workload, by default, submissions will then use the batch size of the most similar [fixed workload](#fixed-workloads) (for example, if there is an ImageNet fixed workload and also a randomized workload with a similarly sized model on similarly sized images, the ImageNet batch size will be used for held-out workloads generated from this randomized workload). -Note that submitters are *not* allowed to modify the *evaluation batch size*, which is set by the benchmarking codebase. However, you can file an issue if you believe that the evaluation batch size of a particular workload is set inappropriately. The working group will review this request and consider adjusting the evaluation batch size in the benchmarking codebase, thus affecting all submitters equally. +To participate, you [**submit a training algorithm**](/README.md#how-to-submit) by implementing a specific set of functions within our API, i.e. the [**submission functions**](#submission-api). All other components, including the model architecture, loss function, and evaluation logic, are fixed. This ensures that any performance gains are directly attributable to your algorithmic innovations. -The **submission functions** are the *batch size getter*, *optimizer state initializer*, *variable update*, *prepare for evaluation function*, and *data selection functions*. The *fixed functions* are the *data augmentation/preprocessing*, *model initialization*, *forward pass*, and *loss function*. The trained model will be evaluated in a separate step that does not call any of the submitted code. - -##### Fixed functions - -With the exception of `_build_input_queue`, submitters can call any of these functions (along with any public function in the provided `Workload` instance) at any time in their submitted functions. - -```python -@property -def step_hint(self): -> int -``` +Submissions can be entered under two distinct rulesets: -- The `step_hint` function gives the number of global steps the baseline algorithm can perform with the `max_runtime` to reach the targets for a workload. The `step_hint` is therefore dependent on the `max_runtime` and the workload. Note that the baseline algorithms may have reached the target in fewer steps than this, but these were the max number of steps the baseline algorithms used for their learning rate schedules. Submitters can use this to help specify learning rate (or other) schedules. +1. **External Tuning Ruleset:** This ruleset permits a limited, automated, parallel hyperparameter search for each workload, where the search space is defined by the submitter but must be the same for all workloads. A submission's workload score uses only the fastest tuning trial to reach the target. +2. **Self-Tuning Ruleset:** This ruleset is for hyperparameter-free or fully autonomous algorithms. All workload adaptations or hyperparameter tuning must be performed by the algorithm "on the clock" during a single training run. -###### Data augmentation and preprocessing +A core tenet of the benchmark is to foster the development of broadly applicable methods. Submissions must be able to generalize and are prohibited from using logic or pre-computed solutions specific to any single workload. -```python -def _build_input_queue( - self, - data_rng: RandomState, - split: str, - data_dir: str, - global_batch_size: int) -> Iterator[Dict[str, Tensor]]: -``` +## Benchmarking Process -- The `_build_input_queue` function will be called to produce the iterator over batches that the submitted data selection function consumes. It is responsible for all data reading, shuffling, repeating, preprocessing, and batching. Note that for Jax this should return an iterator over tensors of shape `(num_devices, per_device_batch_size, ...)`, and for PyTorch this should return tensors of shape `(per_device_batch_size, ...)` (assuming PyTorch's [DDP](https://pytorch.org/docs/stable/notes/ddp.html) is used). +The following sections provide the complete technical specifications of the benchmarking process, starting with what constitutes a [**Submission**](#submissions), followed by the two rulesets handling [**Hyperparameter Tuning**](#tuning-rulesets). The [**Workloads**](#workloads) section outlines the deep learning workloads (i.e. models, datasets, loss functions, etc.) used in the benchmark. Finally, the [**Scoring**](#scoring) section describes the process of computing a submission's final, scalar AlgoPerf score (as well as alternative scoring metrics). -###### Model initialization +### Submissions -```python -def init_model_fn(self, rng: RandomState) -> initial model parameters -``` +A submission to the _AlgoPerf_ benchmark consists of a `submission.py` file that implements a set of Python functions that define your custom training algorithm. This code will be called by the benchmark harness that manages the overall training and evaluation loop. +The core idea is that a submission replaces specific parts of a standard training pipeline with its own logic to train the _AlgoPerf_ workloads to the target performance as quickly as possible, while adhering to the benchmark's rules. -- Unlike in the *Model Track*, this function that initializes the parameters of the model, is fixed. While it can be called by the submission (e.g. to restart the model after a failed training effort) it cannot be changed. +This section details the functions you must implement (the [**Submission API**](#submission-api)), the most important functions and data provided by the benchmark environment ([**fixed functions**](#fixed-functions)), the [**rules to create a valid submission**](#valid-submissions), as well as the [**runtime environment and evaluation procedure**](#runtime-environment-and-evaluation). -###### Forward pass +#### Submission API -```python -def model_fn( - self, - params: ParameterContainer, - augmented_and_preprocessed_input_batch: Tensor, - model_state: ModelAuxiliaryState, - mode: ForwardPassMode, # mode \in {train, eval} - rng: RandomState, - hyperparameters: Hyperparameters, - update_batch_norm: bool, - dropout_rate: float -) -> (logits_output_batch, new_model_state): Tuple[Tensor, ModelAuxiliaryState] -``` +The submission functions are the [`get_batch_size`](#get_batch_size), [`init_optimizer_state`](#init_optimizer_state), [`update_params`](#update_params), [`prepare_for_eval`](#prepare_for_eval), and [`data_selection`](#data_selection) functions. These functions are the only ones that submitters are allowed to modify. +All other functions are [**fixed functions**](#fixed-functions) and contain among other things the `step_hint`, `_build_input_queue`, `init_model_fn`, `model_fn`, or `loss_fn` functions. +Although a submission might access these fixed functions, e.g., to re-initialize the model after a failed training effort, it is not allowed to modify them. +The trained model will be evaluated in a separate step that does not call any of the submitted code. -- `params` is whatever the structure is that contains the (`float32`) model parameters. The naming is overloaded due to having to handle the more object-oriented `PyTorch` style and the functional `JAX` style of development. In the `Flax` library (written in `JAX`), this is typically a nested dictionary of `JAX`/`numpy` arrays, but in `PyTorch` this is the `torch.nn.Model`. -- It is possible that `model_parameters` will be endowed with additional information about the kind of each parameter, e.g. "weights" or "bias" or "batch norm", although `model_fn` does not really need that information we might use the same nested structure elsewhere -- `logits_output_batch` is before the output activation -- `new_model_state` is for batch norm or similar side effects and will only be updated if `update_batch_norm` is set -- `hyperparameters` will contain only dropout rates, which will be used in the models that support it. These can be tuned or will default to documented model-specific values. Note that adding additional dropout would be considered changing the model, which is not allowed, but the tuning of dropout in existing dropout layers can be considered a regularizer, so we allow it. There should be at most two dropout rates in a model (if there are more than two we will reuse the same values). -- `dropout_rate` is used in the model forward pass. If not provided, the workload’s default value is used (see below for the list of defaults). +> 💡 In principle, submissions are allowed to use the available hardware systems in any data- or model-parallel manner they desire, within the constraints of the submission function APIs. However, in practice, model-parallelism may not be possible with the API. Submitters are allowed to access any framework-specific device information necessary to exploit the hardware. -###### Loss function +##### `get_batch_size` ```python -def loss_fn( - self, - # Dense or one-hot labels, or a tuple of (tensor, padding) for speech. - label_batch: Union[Tuple[Tensor, Tensor], Tensor], - logits_batch: Union[Tuple[Tensor, Tensor], Tensor], - mask_batch: Optional[Tensor] = None, - label_smoothing: float = 0.0) -> Dict[str, Tensor] # differentiable +def get_batch_size(workload_name: str) -> int ``` -- Unlike in the *Model Track*, we will specify the loss function name in order to let training algorithms depend on the loss function. It will be one of {**mean squared error**, **cross-entropy**, **CTC**, or **L1 reconstruction error**}. - - The optimizer must work with all values of the enum, which will be provided via a property on the workload object that is provided to all submissions functions. -- The loss function does **not** include regularization. Instead, regularization can be added by the submissions in the `update_params` function. -- The loss function returns a dict {'summed': scalar summed loss, 'n_valid_examples': scalar number of valid examples in batch, 'per_example': 1-d array of per-example losses}. - Note that the returned quantities are not synced across devices; this can be done by the user in the `update_params` function. - -##### Submission functions - -###### Batch size getter +**Purpose:** To specify the training batch size for a given workload. -```python -def get_batch_size(workload_name: str) -> int -``` +- This function allows submitters to define a different batch size for each workload to ensure that the training does not run out of memory. +- For example, in advance, submitters can determine, for each workload, the largest batch size that fits into memory of the [benchmarking hardware](#benchmarking-hardware). +- Called once per workload before training begins. -- Submitters define a specific batch size for each [workload](#workloads). -- For example, in advance, they can determine the largest batch size without running out of memory for each workload. -- For the [held-out workloads](#randomized-workloads), by default, this function will use the `workload_name` of the fixed workload it is based on. -- This does not affect the *evaluation batch size* (i.e. the batch size used during the evaluation phase), which is, by design, a quantity submitters are not allowed to change. +> [!NOTE] +> +> This does not change the _evaluation batch size_ (i.e., the batch size used during the evaluation phase). By design, submitters are not allowed to modify the evaluation batch size, which is set by the benchmarking codebase. However, you can file an issue if you believe that the evaluation batch size of a particular workload is set inappropriately. The working group will review this request and consider adjusting the evaluation batch size in the benchmarking codebase, thus affecting all submitters equally. -###### Optimizer state initializer +##### `init_optimizer_state` ```python def init_optimizer_state( @@ -175,11 +117,12 @@ def init_optimizer_state( ) -> initial_optimizer_state ``` -- Allowed to create state for the optimizer -- Does not involve the initialization for the model parameters, which in the Training Algorithm Track, is considered a fixed function, see [Model initialization](#model-initialization). +**Purpose:** To initialize the optimizer state, i.e., momentum buffers or defining learning rate schedules. + +- It does not involve the [initialization for the model parameters](#fixed-functions), which in this benchmark is considered a fixed function. - The optimizer state is a dictionary (`Dict[str, Any]`). For a PyTorch submission, any value in this dictionary which is a class instance with internal state has to have a `state_dict()` method implemented to be stored correctly at the training checkpoints. -###### Variable update function +##### `update_params` ```python def update_params( @@ -191,30 +134,28 @@ def update_params( batch: Dict[str, Tensor], loss_type: LossType, optimizer_state: OptimizerState, - train_state: Dict[str, Any], eval_results: List[Tuple[int, float]], global_step: int, - rng: RandomState -) -> (updated_optimizer_state, updated_variables, updated_model_state) + rng: RandomState, + train_state: Optional[Dict[str, Any]] = None +) -> (updated_optimizer_state, updated_params, updated_model_state) ``` +**Purpose:** To perform a single training step, i.e., update the model parameters and optimizer state. + +- Inside this function, you will typically call the workload's `loss_fn` and `model_fn` to perform a forward and backward pass to get gradients. + - Uses the `model_fn` of the `workload` in order to decouple the loss from the model so that model outputs (forward passes) can be reused (by storing them in the optimizer state). +- The fixed `init_model_fn` can optionally be called during training, for example, to reinitialize the model after a failed training effort. +- **A call to this function will be considered a step**. The time between a call to this function and the next call to this function will be considered the per-step time. +- A submission can access the elapsed training time and get further information about the evaluation through `train_state`. It may also access the target evaluation metric via the `workload` variable. - `current_param_container` is the same kind of nested structure as used by `model_fn` which constitutes a nested collection of `float32` arrays, each endowed with information about what kind of parameter that array represents stored in a parallel structure of `current_params_types`. - - Parameter kind is one of {"weights", "biases", "embeddings", "conv", "batch norm"}. + - Parameter kind is one of a known list of types, e.g. `{"weights", "biases", "embeddings", "conv_weight", "batch_norm_scale", "batch_norm_bias", ...}`. - `model_state` holds auxiliary state necessary for some models, such as the current batch norm statistics. - The loss function will be one of a small set of known possibilities and the update function is allowed to branch on the `loss_type` enum/name. - The `loss_fn` produces a loss per example and a summed loss (both only for one device), which both can be used. -- Allowed to update state for the optimizer. -- Uses the `model_fn` of the `workload` in order to decouple the loss from the model so that model outputs (forward passes) can be reused (by storing them in the optimizer state). -- The submission can access the elapsed training time and get further information about the evaluation through `train_state`. -- The submission can access the target evaluation metric via the `workload` variable. -- **A call to this function will be considered a step** - - The time between a call to this function and the next call to this function will be considered the per-step time. -- Cannot modify the given hyperparameters in a workload-conditional way (please see the [Valid submission](#valid-submissions) section). This rule is intended to prohibit circumventing the tuning rules by looking up a pre-tuned optimal set of hyperparameters for each workload. It is not intended to prohibit line searches and other similar techniques. -- The fixed `init_model_fn` can optionally be called during training, for example, to reinitialize the model after a failed training effort. -- Cannot replace the model parameters with pre-trained ones. -- Batch norm should work here because the `model_fn` will return updated batch norm moving averages when it is told to with `update_batch_norm`. +- Cannot modify the given hyperparameters in a workload-conditional way (please see the [Valid Submissions](#valid-submissions) section). This rule is intended to prohibit circumventing the tuning rules by looking up a pre-tuned optimal set of hyperparameters for each workload. It is not intended to prohibit line searches and other similar techniques. -###### Prepare for evaluation function +##### `prepare_for_eval` ```python def prepare_for_eval( @@ -228,55 +169,124 @@ def prepare_for_eval( eval_results: List[Tuple[int, float]], global_step: int, rng: RandomState -) -> (updated_optimizer_state, updated_variables, updated_model_state) +) -> (updated_optimizer_state, updated_params, updated_model_state) ``` -- Arguments are the same of `update_param`, with the only exception of `batch`. -- This function is called when a submission is deemed eligible for an evaluation (see [Evluation during training](#evaluation-during-training) section). +**Purpose:** To prepare the model for evaluation, e.g., for swapping model parameters. + +- Arguments are the same as `update_params`, with the only exception of `batch`. +- This function is called when a submission is deemed eligible for an evaluation (see [Evaluation during training](#evaluation-during-training) section). - The call to `prepare_for_eval` is timed and its runtime accumulates to the overall submission time. - - The returned model parameters are evaluated on the validation and test sets, provided that the accumulated submission time does not exceed the maximum runtime after this function call. + - The returned model parameters are evaluated on the validation and test sets, provided that the accumulated submission time does not exceed the maximum runtime after this function call (else the evaluation is skipped and the run is terminated). - This API supports Polyak averaging and similar methods that implement moving averages of model parameters. - Allowed to update model state and model parameters. - Allowed to update state for the optimizer. -- Cannot replace the model parameters with pre-trained ones. -###### Data selection +##### `data_selection` ```python def data_selection( workload: Workload, - input_queue: Iterator[Tuple[Tensor, Tensor]], + input_queue: Iterator[Dict[str, Any]], optimizer_state: OptimizerState, current_param_container: ParameterContainer, + model_state: ModelAuxiliaryState, hyperparameters: Hyperparameters, global_step: int, rng: RandomState ) -> Dict[str, Tensor] ``` +**Purpose:** To select a subset of the training data for the next training step. + - `input_queue` can yield up to the number of elements in the training dataset - Want to allow for submitters to construct their own data batches from the dataset - Submissions are allowed to arbitrarily modify the input examples, as long as the modifications are sufficiently generic to be applicable to any workload -- This is only called on the training inputs. **No submitted code will be called at eval in the training track.** -- This allows for any of the following methods: - - Data echoing - - Curriculum learning - - Bootstrapping - - Biased sampling (based on loss values, so need to store the forward pass in the `optimizer_state`, potentially forward pass of a cheaper proxy model) - - Submissions need batching control +- This is only called on the training inputs. **No submitted code will be called during evaluation.** +- Examples of data selection methods include _data echoing_, _curriculum learning_, _bootstrapping_, or _biased sampling_ (based on loss values, so need to store the forward pass in the `optimizer_state`, potentially forward pass of a cheaper proxy model). -#### Evaluation during training +##### Fixed Functions -In general, with noisy, non-deterministic training, evaluation frequency can affect training time measurements as more "bites of the apple" potentially allows the training code to exploit instability. We also want to discourage submissions from complicated and unrealistic logic that attempts to guess when training is close to complete and increases the evaluation rate, while not producing a well-sampled training curve at the start of training. Simply allowing submissions complete freedom over evaluation frequency encourages competitors to work to minimize the number of evaluations, which distracts from the primary goal of finding better training algorithms. +Any function that is not part of the [**Submission API**](#submission-api) (and thus a submission function) is considered a fixed function, which submitters are **not** allowed to modify. +Below, we describe some of the fixed functions to provide a better understanding of the _AlgoPerf_ benchmark API. With the exception of `_build_input_queue`, submitters can call any of these functions (along with any public function in the provided `workload` instance) at any time in their submission functions. + +###### Step hint + +```python +@property +def step_hint(self) -> int +``` + +- The `step_hint` function gives the number of global steps the baseline algorithm can perform within the `max_runtime` on a workload. The `step_hint` is therefore dependent on the workload and its (current) `max_runtime`. Note that the baseline algorithms may have reached the target in fewer steps than this, but these were the max number of steps the baseline algorithms used for their learning rate schedules. Submitters can use this to help specify learning rate (or other) schedules. +- The `step_hint` is only a hint. There is no need to use it at all. However, it is often more convenient, e.g. to define the learning rate schedule in terms of the number of steps (instead of the runtime). + +###### Data augmentation and preprocessing + +```python +def _build_input_queue( + self, + data_rng: RandomState, + split: str, + data_dir: str, + global_batch_size: int +) -> Iterator[Dict[str, Tensor]] +``` + +- The `_build_input_queue` function will be called to produce the iterator over batches that the submitted data selection function consumes. It is responsible for all data reading, shuffling, repeating, preprocessing, and batching. Note that for JAX this should return an iterator over tensors of shape `(num_devices, per_device_batch_size, ...)`, and for PyTorch this should return tensors of shape `(per_device_batch_size, ...)` (assuming PyTorch's [DDP](https://pytorch.org/docs/stable/notes/ddp.html) is used). + +###### Model initialization -Submissions are eligible for an untimed eval every `eval_period` seconds. Before proceeding to evaluation, the submission can prepare the model through a call to `prepare_for_eval`, effectively modifying the model parameters and state as well as the the optimizer state. Any additional evaluations performed by the submission code count against the runtime for scoring. -The harness that runs the submission code will attempt to eval every `eval_period` seconds by checking between each submission step (call of `update_params`) whether it has been at least `eval_period` seconds since that last eval, if so, the submission is given the possibility to prepare for evaluation (through a timed call to `prepare_for_eval`). If the accumulated runtime does not exceed the maximum allowed runtime after the preparation step, the clock is paused, and the submission is evaluated. This means that if calls to `update_params` typically take a lot more than `eval_period` seconds, such submissions will not receive as many untimed evals as a submission that had an `update_params` function that took less time. However, for appropriate settings of `eval_period`, we expect this to be quite rare. Submissions are always free to restructure their `update_params` code to split work into two subsequent steps to regain the potential benefits of these untimed model evaluations. For each workload, the `eval_period` will be set such that the total evaluation time is roughly between 10% and 20% of the total training time for the target-setting runs. +```python +def init_model_fn(self, rng: RandomState) -> ModelInitState +``` + +- This function initializes the parameters of the model. While it can be called by the submission (e.g. to restart the model after a failed training effort) it cannot be changed. -#### Valid submissions +###### Forward pass + +```python +def model_fn( + self, + params: ParameterContainer, + augmented_and_preprocessed_input_batch: Tensor, + model_state: ModelAuxiliaryState, + mode: ForwardPassMode, # mode \in {train, eval} + rng: RandomState, + update_batch_norm: bool, + dropout_rate: float +) -> (logits_output_batch, new_model_state): Tuple[Tensor, ModelAuxiliaryState] +``` + +- `params` is whatever the structure is that contains the (`float32`) model parameters. The naming is overloaded due to having to handle the more object-oriented `PyTorch` style and the functional `JAX` style of development. In the `Flax` library (written in `JAX`), this is typically a nested dictionary of `JAX`/`numpy` arrays, but in `PyTorch` this is the `torch.nn.Model`. +- It is possible that `model_parameters` will be endowed with additional information about the kind of each parameter, e.g. "weights" or "bias" or "batch norm", although `model_fn` does not really need that information we might use the same nested structure elsewhere. +- `logits_output_batch` is before the output activation. +- `new_model_state` is for batch norm or similar side effects and will only be updated if `update_batch_norm` is set. +- `dropout_rate` is used in the model forward pass for models that support it. These can be tuned or will default to documented model-specific values (see the [workloads table](#workloads) for the list of defaults). Note that adding additional dropout would be considered changing the model, which is not allowed, but the tuning of dropout in existing dropout layers can be considered a regularizer, so we allow it. There should be at most two dropout rates in a model (if there are more than two we will reuse the same values). The `dropout_rate` can be changed during the training process. + +###### Loss function + +```python +def loss_fn( + self, + # Dense or one-hot labels, or a tuple of (tensor, padding) for speech. + label_batch: Union[Tuple[Tensor, Tensor], Tensor], + logits_batch: Union[Tuple[Tensor, Tensor], Tensor], + mask_batch: Optional[Tensor] = None, + label_smoothing: float = 0.0 +) -> Dict[str, Tensor] # differentiable +``` + +- The loss function does **not** include regularization. Instead, regularization can be added by the submissions in the `update_params` function. +- The loss function returns a dict `{'summed': scalar summed loss, 'n_valid_examples': scalar number of valid examples in batch, 'per_example': 1-d array of per-example losses}`. + - Note that the returned quantities are not synced across devices; this can be done by the user in the `update_params` function. +- Each workload uses one of the following loss functions: {_mean squared error_, _cross-entropy_, _CTC_, or _L1 reconstruction error_}. + - Your submission must work with all these loss types. We provide the loss type via a workload property in order to let training algorithms depend on the loss function. + +#### Valid Submissions The intention of this benchmark is to identify training algorithm submissions that will be broadly applicable and effective in practical scenarios without customization to the specific [workload](#workloads) (model, dataset, and loss function). Generally useful training algorithms can train models faster and thus require less compute resources, decreasing the cost of machine learning. We want to discourage all submissions that sidestep the purpose of this benchmark. We welcome creative ideas and novel research. Therefore, the API aims to allow a wide variety of submissions. However, in some cases, routines that would be allowed in principle might not be practically feasible to express in the provided framework. -Submissions that specialize to the specific workloads in the benchmark and have not been implemented in a way that plausibly generalizes to novel workloads are prohibited. +A valid submission must implement **general-purpose training logic** that is expected to work on unseen workloads **without** workload-specific modifications or precomputed lookups. In order to help clarify which submissions are [allowed](#allowed-submissions) and [disallowed](#disallowed-submissions), we described a few examples below. Two essential questions can help provide a general guideline for whether a submission is allowed or not: 1. What **information** is being used by the submission? @@ -316,7 +326,7 @@ Submissions can also be based on learned training algorithms.
Examples: -- Submission are allowed to learn the update rule of the training method. +- Submissions are allowed to learn the update rule of the training method. - In the [self-tuning ruleset](#self-tuning-ruleset), submissions could try out a learned list of hyperparameters.
@@ -333,289 +343,344 @@ Submissions can use additional software dependencies provided they have the inte ##### Disallowed submissions +Submissions must rely on new algorithmic or mathematical ideas and concepts, and must not use software engineering approaches in order to increase primitive operations in PyTorch, JAX, their dependencies, the operating systems, or the hardware. Submissions may use public APIs in JAX and PyTorch from within the submission function APIs, but may not use APIs to optimize the internals of primitive operations and/or standard dependencies to benefit any submission. + Submissions are not allowed to circumvent the tuning rules by looking up the result of an offline computation that was performed ahead of time.
Examples: - Submissions are not allowed to look up (pre-trained) model parameters. -- Computing the optimal hyperparameters for every fixed workload offline and having the submission look up those pre-computed values (and finding the closest fixed workload for a held-out workload) is not allowed. In contrast, finding and hard-coding a single good setting of the hyperparameters that works well across all the workloads simultaneously would be allowed. +- Computing the optimal hyperparameters for every workload offline and having the submission look up those pre-computed values is not allowed. In contrast, finding and hard-coding a single good setting of the hyperparameters that works well across all the workloads simultaneously would be allowed. - Submissions are not allowed to adjust the hyperparameter search spaces for the external tuning ruleset, such that it differs between the workloads.

-Submissions are not allowed to detect the particular workload (irrespective of which information they use to this end) in order to use settings that are specified for individual workloads. This would result in highly specific behavior that isn't generally useful. This also extends to learned approaches that ultimately detect specific workloads. In general, all else being equal, if some submission was written that was extremely effective on a small set of the workloads (and far worse on the rest) and another submission with the opposite performance pattern, we would prefer both submissions to be submitted and tested on **all** workloads. +Submissions may not identify (directly or indirectly) the specific benchmark workload to select special-cased logic or hyperparameters; learned detectors that end up selecting workload-specific behavior are equally disallowed. This would result in highly specific behavior that isn't generally useful. In general, all else being equal, if some submission was written that was extremely effective on a small set of the workloads (and far worse on the rest) and another submission with the opposite performance pattern, we would prefer both submissions to be submitted and tested on **all** workloads.
Examples: - A hard-coded switching of the update rule based on the workload is not allowed, e.g. using Adam for RNNs and SGD with momentum on CNNs. Although submissions can specialize for certain layer types in generic ways, they should not uniquely identify a model or dataset. In other words, if there are two workloads A and B that both have convolutional layers and fully connected layers the submission shouldn't detect whether it is dealing with A or B specifically and choose Adam for one and SGD with momentum for the other. However, if the updates for all parameters of convolutional layers always used SGD with momentum and the updates for all other layers always used Adam and a workload with both types of layers had mixed updates, that would be fine. -It is also allowed to make the update rule part of the (external) hyperparameter tuning or determine the optimal update rule during the run, i.e. while "on-the-clock". -- Submissions are not allowed to look up learning rate schedules that are only utilized for specific subsets of the workloads. It is allowed to use one general learning rate schedule or dynamically adapt the learning rate based on general information such as curvature. + It is also allowed to make the update rule part of the (external) hyperparameter tuning or determine the optimal update rule during the run, i.e. while "on-the-clock". +- Submissions are not allowed to look up learning rate schedules that are only utilized for specific subsets of the workloads. It is allowed to use one general learning rate schedule, to dynamically adapt the learning rate based on general information such as curvature, or to select the learning rate schedule as part of the (external) hyperparameter tuning.

-It is not allowed to compute any kind of pairwise metrics between the fixed workloads and the held-out workloads. +Valid submissions must rely on new algorithmic or mathematical ideas and should not use software engineering approaches to speed up primitive operations in `PyTorch`, `JAX`, their dependencies, the operating system, or the hardware. We recognize that the way a method is implemented will impact its performance in the benchmark. It is generally acceptable to make clever, judicious, and efficient use of public APIs in `JAX` and/or `PyTorch` from within the submission function APIs. It is not acceptable to use these APIs to optimize the internals of primitive operations and standard dependencies in ways that could generally benefit any submission.
Examples: -- On a held-out workload, submissions are not allowed to find the nearest neighbor among the fixed workloads to set any hyperparameter. +- Submissions **are allowed** to use `CUDA` streams to schedule operations, e.g., transferring data between CPU and GPU, or among GPUs, while performing other computations. +- Submissions **are not allowed** to use `CUDA` streams or asynchronous operations (e.g., spawning additional threads) to perform additional computations that run during the [untimed evaluations](#evaluation-during-training). +- Submissions **are not allowed** to use faster GPU kernels than other submitters by writing their own, using `TVM`, or using a different version of `cuDNN`/`cuBLAS`. +- Submissions **are not allowed** to skip or reduce system or framework overhead, such as modifying `JAX` to skip internal steps like pytree flattening/unflattening. +- Submissions **are not allowed** to introduce new compiler optimizations, such as modifying `XLA` to perform more or less kernel fusion.
-
-Valid submissions must rely on new algorithmic or mathematical ideas and should not use software engineering approaches to speed up primitive operations in `PyTorch`, `JAX`, their dependencies, the operating system, or the hardware. We recognize that the way a method is implemented will impact its performance in the benchmark. It is generally acceptable to make clever, judicious, and efficient use of public APIs in `JAX` and/or `PyTorch` from within the submission function APIs. It is not acceptable to use these APIs to optimize the internals of primitive operations and standard dependencies in ways that could generally benefit any submission. +#### Runtime Environment and Evaluation -
-Examples: +##### Evaluation during training -- Submissions are allowed to use `CUDA` streams to schedule operations, e.g., transfering data between CPU and GPU, or among GPUs, while performing other computations. -- Submissions are not allowed to use `CUDA` streams or asynchronous operations (e.g., spawning additional threads) to perform additional computations that run during the [untimed evaluations](#evaluation-during-training). -- Submissions are not allowed to use faster GPU kernels than other submitters by writing their own, using `TVM`, or using a different version of `cuDNN`/`cuBLAS`. -- Submissions are not allowed to skip or reduce system or framework overhead, such as modifying `JAX` to skip internal steps like pytree flattening/unflattening. -- Submissions are not allowed to introduce new compiler optimizations, such as modifying `XLA` to perform more or less kernel fusion. +In general, with noisy, non-deterministic training, evaluation frequency can affect training time measurements as more "bites of the apple" potentially allows the training code to exploit instability. We also want to discourage submissions from complicated and unrealistic logic that attempts to guess when training is close to complete and increases the evaluation rate, while not producing a well-sampled training curve at the start of training. Simply allowing submissions complete freedom over evaluation frequency encourages competitors to work to minimize the number of evaluations, which distracts from the primary goal of finding better training algorithms. -
+Submissions are eligible for an untimed evaluation every `eval_period` seconds. Before proceeding to evaluation, the submission can prepare the model through a call to `prepare_for_eval`, effectively modifying the model parameters/state as well as the optimizer state. Any (optional) additional evaluations performed by the submission code count against the runtime for scoring. +The harness that runs the submission code will attempt to evaluate every `eval_period` seconds by checking between each submission step (call of `update_params`) whether it has been at least `eval_period` seconds since that last evaluation. If so, the submission is given the possibility to prepare for evaluation (through a timed call to `prepare_for_eval`). If the accumulated runtime does not exceed the maximum allowed runtime after the preparation step, the clock is paused, and the submission is evaluated. This means that if calls to `update_params` typically take a lot more than `eval_period` seconds, such submissions will not receive as many untimed evaluations as a submission that had an `update_params` function that took less time. However, for appropriate settings of `eval_period`, we expect this to be quite rare. Submissions are always free to restructure their `update_params` code to split work into two subsequent steps to regain the potential benefits of these untimed model evaluations. For each workload, the `eval_period` will be set such that the total evaluation time is roughly between 10% and 20% of the total training time for the target-setting runs. -##### Software dependencies +##### Software Dependencies -We require submissions to use specific versions of `PyTorch`/`JAX` as well as additional dependencies in order to facilitate fair comparisons. Submitters must build on top of these provided software packages, which might be provided as a `Docker` container. Additional dependencies can be added as long as they include a comment describing what was added and why. Submitters are free to add dependencies that support new algorithmic and mathematical ideas but they should not circumvent the intention of the benchmark to measure training speedups due to new training methods. For example, software engineering techniques that lead to faster implementations of existing software, e.g. using newer versions of `PyTorch` or `JAX`, are not allowed and these are described in more detail in the [Disallowed submissions](#disallowed-submissions) section. +If your submission will have any software dependencies, you must create a `requirements.txt` file in the `/submission` directory. This file must clearly list all software dependencies your submission requires in order to be a valid submission. The file must be "pip-readable" (the dependencies listed can be installed via the `pip install -r requirements.txt` command). You may not modify the package versions of the software dependencies used by the benchmarking codebase, including using a different version of libraries such as PyTorch or JAX from those specified in the benchmark. -##### Environment variables +We require submissions to use specific versions of `PyTorch`/`JAX` as well as additional dependencies in order to facilitate fair comparisons. Submitters must build on top of these pinned software packages. Additional dependencies can be added as long as they include a comment describing what was added and why. Submitters are free to add dependencies that support new algorithmic and mathematical ideas but they should not circumvent the intention of the benchmark to measure training speedups due to new training methods. For example, software engineering techniques that lead to faster implementations of existing software, e.g. using newer versions of `PyTorch` or `JAX`, are not allowed and these are described in more detail in the [Disallowed submissions](#disallowed-submissions) section. -The benchmark codebase sets environment variables, and submitters are not permitted to modify (or add) environment variables for the software dependencies. However, if you believe some of these variables are not optimally set, you can suggest changes, e.g. by creating an issue. The working group will evaluate your suggestions and consider making adjustments if necessary. This ensures that all submissions are equally affected by the environment variables and maintains the competition's primary focus on algorithmic improvements. +##### Environment Variables -### Tuning +The benchmark codebase sets environment variables, and submitters are not permitted to modify (or add) environment variables for the software dependencies. However, if you believe a setting is sub-optimal, open an issue with justification; the working group may adjust it. This ensures that all submissions are equally affected by the environment variables and maintains the competition's primary focus on algorithmic improvements. -Tuning will be substantially different for the [external](#external-tuning-ruleset) and the [self-tuning ruleset](#self-tuning-ruleset) and the individual specifications for each will be described in the following. +### Tuning Rulesets -#### External tuning ruleset +Tuning will be substantially different for the [**external**](#external-tuning-ruleset) and the [**self-tuning ruleset**](#self-tuning-ruleset) and the individual specifications for each will be described in the following. -For each workload, the hyperparameters are tuned using $O=5$ tuning **trials**. To estimate the variance of the results, this tuning will be repeated for $S=5$ **studies**, for a total of $S\cdot O = 25$ different hyperparameter settings. The submitters will provide a workload-agnostic search space and the working group will then return $25$ hyperparameters settings obtained using [(quasi)random search](https://arxiv.org/abs/1706.03200). The working group will also randomly partition these $25$ trials into $5$ studies of $5$ trials each. In lieu of independent samples from a search space, submissions can instead supply a fixed list of $5$ hyper-parameter points that will be sampled without replacement. +#### External Tuning Ruleset -In each trial, the tuning trial with the fastest training time to achieve the *validation target* is determined among the $O=5$ hyperparameter settings. For scoring, we use this required training time to reach the *validation targets* of those $5$ selected runs. The median of these $5$ per-study training times will be the final training time for the submission on this workload and is used in the scoring procedure (see the "[Scoring submissions](#scoring)" section). Runs that do not reach the target performance of the evaluation metric have an infinite time. Submissions are always free to perform additional self-tuning while being timed. +For every workload, **$5$ tuning _trials_** are run, and this tuning process is **repeated in $3$ independent _studies_** to capture variance, resulting in $15$ runs overall. +Submitters have to provide a _workload-agnostic search space_, via a `tuning_search_space.json` file. +During scoring, we draw $15$ hyperparameter configurations from this search space using [(quasi)random search](https://arxiv.org/abs/1706.03200) and randomly assign them to the $3$ studies with each $5$ trials. +Instead of independent samples from a search space, submitters can also provide a fixed list of $5$ hyperparameter points, which will be sampled without replacement for each study. -#### Self-tuning ruleset +Within each study, we select the fastest trial that reaches the validation target. The median of the three per-study best times is the submission's official _per-workload score_. These $8$ _per-workload runtimes_ are used in the scoring procedure (see the [**Scoring submissions**](#scoring) section). Trials that do not reach the target within `max_runtime` receive $\infty$, (which participates in the median). +Submissions may also perform on-the-clock self-tuning during timed training. -Submissions to this ruleset are not allowed to have user-defined hyperparameters. This ruleset allows both submissions that use the same hyperparameters for all workloads, including the randomized ones (e.g. Adam with default parameters), as well as submissions that perform inner-loop tuning during their training run (e.g. SGD with line searches). +> [!IMPORTANT] +> +> - **Trial**: One training run, with a fixed hyperparameter configuration until the target or `max_runtime` was reached. The first time the validation target is reached in a trial is denoted $t_{i,j}$ (a miss scores $\infty$). +> - **Study**: A set of $5$ trials, each run with distinct hyperparameter points. The studies are independent and capture variance. The study's score is the **fastest** (minimum) time among its trials. +> - **Per-Workload Runtime**: The per-workload runtime of a submission is given by the median across the per-study scores, i.e., $t_{s,w} = median_{j=1..3} \left( \min_{i=1..5} (t_{i,j}) \right)$, with $t_{i,j}$ the score of trial $i$ in study $j$, i.e. -Submissions will run on one instance of the [benchmarking hardware](#benchmarking-hardware). As always, submissions are allowed to perform inner-loop tuning (e.g. for their learning rate) but the tuning efforts will be part of their score. A submission will run *S=5* times and its score will be the median time to reach the target evaluation metric value on the validation set. To account for the lack of external tuning, submissions have a longer time budget to reach the target performance. Compared to the [external tuning ruleset](#external-tuning-ruleset), the `max_runtime` is $1.5$ times longer. Runs that do not reach the target performance of the evaluation metric within this allotted time budget have an infinite time. -### Workloads +#### Self-Tuning Ruleset -For the purposes of the Training Algorithm Track, we consider a workload the combination of a `dataset`, `model`, `loss_fn`, along with a target that is defined over some evaluation metric. E.g., ResNet50 on ImageNet using the cross-entropy loss until a target error of 22.6% on the validation set has been reached, would constitute a workload. The evaluation metric, in this example the misclassification error rate, is directly implied by the dataset/task. +Submissions under this ruleset are not allowed to expose user-defined hyperparameters. +Instead, submissions can either apply one "default" hyperparameter configuration for all workloads (e.g., Adam with default settings), or perform inner-loop tuning during their training run (e.g., SGD with line searches). +All workload adaptations, e.g. inner-loop tuning, will be part of the submission's score. -Submissions will be scored based on their performance on the [fixed workload](#fixed-workloads). However, additionally submissions must also perform resonably well on a set of [held-out workloads](#randomized-workloads) in order for their score on the fixed workload to count (for full details see the [Scoring](#scoring) section). These held-out workloads will be generated after the submission deadline, but their randomized generating process is publicly available (see "[Randomized workloads](#randomized-workloads)" section). +For each workload, a submission will run for **$3$ independent studies**, and the _per-workload score_ is the median time to reach the validation target, i.e., $t_{s,w} = median_{j=1..3} \left(t_{j}\right)$. +To account for the lack of external tuning, submissions have a longer time budget to reach the target performance. +Compared to the [**external tuning ruleset**](#external-tuning-ruleset), the `max_runtime` is $1.5\times$ longer (i.e. multiply the `max_runtimes` from the [**workload overview table**](#workloads) by $1.5$). +As in the [**external tuning ruleset**](#external-tuning-ruleset), any run that fails to achieve the target within this window is assigned an infinite runtime. -Furthermore, a less computationally expensive subset of the fixed workloads is collected with the [qualification set](#qualification-set). Submitters without enough compute resources to self-report on the full set of fixed and held-out workloads can instead self-report on this smaller qualification set. Well-performing submissions can thereby qualify for computational resources provided by sponsors of the benchmark to be scored on the full benchmark set. +### Workloads -NOTE: Submitters are no longer required to self-report results for AlgoPerf competition v0.5. +For the purposes of the _AlgoPerf: Training Algorithms_ benchmark, we consider a workload the combination of a `dataset`, `model`, `loss_fn`, along with a `target` that is defined over some evaluation `metric`. E.g., `ResNet-50` on `ImageNet` using the `cross-entropy` loss until a target `error` of `22.6%` on the validation set has been reached, would constitute a workload. -#### Fixed workloads +The _AlgoPerf: Training Algorithms_ benchmark contains a diverse set of $8$ workloads spanning tasks such as image classification, machine translation, speech recognition, or other typical machine learning tasks. For a single task and dataset there might be multiple models and therefore multiple workloads. As a rough guideline, the entire set of workloads was designed to have a combined runtime of very roughly $100$ hours on the [**benchmarking hardware**](#benchmarking-hardware). -The fixed workloads are fully specified with the call for submissions. They contain a diverse set of tasks such as image classification, machine translation, speech recognition, or other typical machine learning tasks. For a single task there might be multiple models and therefore multiple fixed workloads. The entire set of fixed workloads should have a combined runtime of roughly 100 hours on the [benchmarking hardware](#benchmarking-hardware). +The eight _AlgoPerf Workloads_ are: -The currently eight fixed workloads are: +| | **Task** | **Dataset** | **Model** | **Loss** | **Metric** | Validation
**Target** | Test
**Target** | Max
**Runtime**
_(in seconds)_ | Default
**Dropout**
Value | +| ----- | ----------------------------- | ----------- | ----------- | -------- | ---------- | ------------------------ | ------------------ | ------------------------------------- | ------------------------------------------- | +| **1** | Clickthrough rate prediction | Criteo 1TB | DLRMsmall | CE | CE (↓) | 0.123735 | 0.126041 | 7,703 | 0 | +| **2** | MRI reconstruction | fastMRI | U-Net | L1 | SSIM (↑) | 0.723653 | 0.740633 | 4,430 | 0 | +| **3** | Image classification | ImageNet | ResNet-50 | CE | ER (↓) | 0.22569 | 0.3440 | 66,159 | None | +| **4** | | | ViT | CE | ER (↓) | 0.22691 | 0.3481 | 69,768 | 0 | +| **5** | Speech recognition | LibriSpeech | Conformer | CTC | WER (↓) | 0.085884 | 0.052981 | 58,015 | 0.1 (`input`, `attn_res`, `ff_res`); else 0 | +| **6** | | | DeepSpeech | CTC | WER (↓) | 0.119936 | 0.074143 | 44,405 | 0.1 (`input`, `ff`); `JAX CudnnLSTM`: 0 | +| **7** | Molecular property prediction | OGBG | GNN | CE | mAP (↑) | 0.28098 | 0.268729 | 12,011 | 0.1 | +| **8** | Translation | WMT | Transformer | CE | BLEU (↑) | 30.8491 | 30.7219 | 43,336 | 0.1 (`main`, `attn`) | -| | **Task** | **Dataset** | **Model** | **Loss** | **Metric** | Validation
**Target** | Test
**Target** | Maximum
**Runtime**
(in secs) | -|------------|-------------------------------|-------------|-------------------------|----------|------------|--------------------------|----------------------|------------------------| -| **1** | Clickthrough rate prediction | Criteo 1TB | DLRMsmall | CE | CE | 0.123735 | 0.126041 | 7,703 | -| **2** | MRI reconstruction | fastMRI | U-Net | L1 | SSIM | 0.723653 | 0.740633 | 4,430 | -| **3
4** | Image classification | ImageNet | ResNet-50
ViT | CE | ER | 0.22569
0.22691 | 0.3440
0.3481 | 66,159
69,768 | -| **5
6** | Speech recognition | LibriSpeech | Conformer
DeepSpeech | CTC | WER | 0.085884
0.119936 | 0.052981
0.074143 | 58,015
44,405 | -| **7** | Molecular property prediction | OGBG | GNN | CE | mAP | 0.28098 | 0.268729 | 12,011 | -| **8** | Translation | WMT | Transformer | CE | BLEU | 30.8491 | 30.7219 | 43,336 | +> [!NOTE] +> Notes on the default dropout column: +> +> - `None` indicates that the model does not use dropout. +> - `0` or `0.1` indicates that the model uses dropout with a default value of 0.0 or 0.1, respectively. +> - `0.1 (main, attn)` indicates that the model uses dropout with a default value of 0.1 for the main `dropout_rate` and the `attention_dropout_rate`. +> - `0.1 (input, attn_res, ff_res) else 0` indicates that the model uses dropout with a default value of 0.1 for `input_dropout_rate`, `attention_residual_dropout_rate`, and `feed_forward_residual_dropout_rate` and use a default value of 0 for all other dropout rates. +> - `0.1 (input, ff) else 0; JAX CudnnLSTM: 0` indicates that the model uses dropout with a default value of 0.1 for `input_dropout_rate` and `feed_forward_dropout_rate`. For JAX models, the `dropout_rate` is set to 0.0 for the `CudnnLSTM` class. +> +> Dropout defaults are used if not overridden by the submission. + +#### Recommended Qualification Set + +Because the full _AlgoPerf: Training Algorithms_ benchmark is computationally quite expensive, we also provide a recommendation for a cheaper variant, the _qualification set_. +This _qualification set_ excludes both _ImageNet_ workloads, both _LibriSpeech_ workloads, and the _fastMRI_ workload, leaving **_Criteo 1TB_, _OGBG_, and _WMT_**. +Together, they run in roughly $24$ hours on the [**benchmarking hardware**](#benchmarking-hardware). +To further reduce computational costs, the [**external tuning ruleset**](#external-tuning-ruleset) uses **only one study** (instead of the proposed $3$) on the qualification set. The [**self-tuning ruleset**](#self-tuning-ruleset) will keep the $3$ studies because it is less costly. -Default Dropout Values for Different Workloads: +> [!NOTE] +> +> The "qualification set" was originally designed as a cheaper benchmark that allowed resource-constrained teams to prove themselves and "qualify" for sponsored compute for the full benchmark. Self-reporting is now optional, but the subset still serves as a cheaper performance estimate, so we're keeping it as a recommendation, including the (historical) name. -| Workload | Dropout Values | -|------------------------|------------------------------------------------------------------------------------------------------| -| criteo 1tb | dropout_rate: 0.0 | -| fastmri | dropout_rate: 0.0 | -| imagenet_resnet | dropout not used | -| imagenet_vit | dropout_rate: 0.0 | -| librispeech_conformer | attention_dropout_rate: 0.0
attention_residual_dropout_rate: 0.1
conv_residual_dropout_rate: 0.0
feed_forward_dropout_rate: 0.0
feed_forward_residual_dropout_rate: 0.1
input_dropout_rate: 0.1 | -| librispeech_deepspeech | input_dropout_rate: 0.1
feed_forward_dropout_rate: 0.1
(Only for JAX - dropout_rate in CudnnLSTM class: 0.0) | -| ogbg | dropout_rate: 0.1 | -| wmt | dropout_rate: 0.1
attention_dropout_rate: 0.1 | +### Scoring -#### Randomized workloads +Submissions are scored based on the training time needed to reach the target performance on each workload's validation set. +The target metric may match the loss function or use another workload-specific metric, such as error rate or BLEU score. +See the [**workload overview table**](#workloads) for the targets and metrics of each workload and the [**Defining target performance**](#defining-target-performance-and-max_runtime) section for how they were determined. +The overall ranking is then determined by the scalar _AlgoPerf Benchmark Score_, which summarizes the _per-workload_ runtimes across all [**workloads**](#workloads), using integrated [**performance profiles**](#algoperf-benchmark-score-via-integrated-performance-profiles), as explained below. -In addition to the [fixed and known workloads](#fixed-workloads), there will also be randomized workloads in our benchmark. These randomized workloads will introduce minor modifications to a fixed workload (e.g. small model changes). The exact instances of these randomized workloads will only be created after the submission deadline and are thus unknown to both the submitters as well as the benchmark organizers. The instructions for creating them, i.e. providing a set or distribution of workloads to sample from, will be defined by this working group and made public with the call for submissions, to allow the members of this working group to submit as well as ensure that they do not possess any additional information compared to other submitters. We will refer to the unspecific workloads as *randomized workloads*, e.g. the set or distribution. The specific instance of such a randomized workload we call a *held-out workload*. That is, a held-out workload is a specific sample of a randomized workload that is used for one iteration of the benchmark. While we may reuse randomized workloads between iterations of the benchmark, new held-out workloads will be sampled for each new benchmark iteration. +> [!NOTE] +> +> The training time of a submission includes the compilation times for computation graphs and ops that could happen just-in-time during training; all our benchmarks should be fast enough to compile so as not to dramatically impact overall performance. -The held-out workloads function similarly to a holdout test set discouraging submissions that overfit to the [fixed and known workloads](#fixed-workloads). After the submission deadline, a third party will draw samples from the randomized workloads (e.g. from the set or the distribution) to generate a specific set of held-out workloads. The validation and test targets on each held-out workload will be defined using the [same protocol as the fixed workloads](#defining-target-performance) (with the only change being that only two target-setting training algorithms are used instead of four, to save computational resources) using the same training time budget as the fixed workload they are based on. +
-Modifications could, for example, include changing the number of layers or units (drawn from an interval), swapping the activation function (drawn from a set of applicable functions), or using different data augmentations (drawn from a list of possible pre-processing steps). The sample space should be wide enough to discourage submitters from simply trying them all out, but at the same time should be restricted enough to produce realistic workloads with acceptable achievable performances. +> [!NOTE] +> +> The training time until the _test set target_ was reached is not used in the scoring procedure but might be used for additional analysis of the competition results. -In the first iteration of this benchmark, we manually designed three different workloads variants for each fixed workload. The variants are designed such that they achieve a comparable performance to the fixed workload and that they might require different hyperparameters to achieve this performance. After the submission deadline, one held-out workload will be sampled for each dataset. +#### AlgoPerf Benchmark Score via Integrated Performance Profiles -Our scoring procedure uses the held-out workloads only to penalize submissions that can't handle the introduced modifications (see the [Scoring](#scoring) section for further details). +We will aggregate the _per-workload training times_ of a submission across all workloads using [Performance Profiles](http://www.argmin.net/2018/03/26/performance-profiles/) (originally from [Dolan and Moré](https://arxiv.org/abs/cs/0102001)). Below we surface several relevant definitions from their work for easier readability, before explaining how we integrate the performance profiles to reach a scalar benchmark score that will be used for ranking submissions. -#### Qualification set +_Notation:_ We have a set $\mathcal{S} = \{s_1, s_2, \dots, s_k\}$ of in total $k$ submissions that we evaluate on a set of $n$ workloads: $\mathcal{W} = \{w_1, w_2, \dots, w_n\}$. For each submission $s$ and each workload $w$ we have a _per-workload runtime_ $t_{s,w} \in [0,\infty)$. This is the median time it took the submission to reach the validation target performance on this particular workload. -NOTE: Submitters are no longer required to self-report results for AlgoPerf competition v0.5. +##### Computing performance ratios -The qualification set is designed for submitters that may not have the compute resources to self-report on the full set of [fixed](#fixed-workloads) and [held-out workloads](#randomized-workloads). They may instead self-report numbers on this smaller qualification set. The best-performing submissions may then qualify for compute sponsorship offering a free evaluation on the full benchmark set and therefore the possibility to win [awards and prizes](/COMPETITION_RULES.md#prizes). +For all workloads and submissions, we first compute their performance ratio $r$, which is defined for a particular submission $\bar{s}$ and a particular workload $\bar{w}$ to be: -The qualification set consists of the same [fixed workloads](#fixed-workloads) as mentioned above, except for both workloads on *ImageNet*, both workloads on *LibriSpeech*, and the *fastMRI* workload. The remaining three workloads (*WMT*, *Criteo 1TB*, and *OGBG*) form the qualification set. There are no [randomized workloads](#randomized-workloads) in the qualification set. The qualification set of workloads aims to have a combined runtime of roughly 24 hours on the [benchmarking hardware](#benchmarking-hardware). +$$r_{\bar{s},\bar{w}} = \frac{t_{\bar{s},\bar{w}}}{\min_{s \in \mathcal{S}} t_{s,\bar{w}}} \in [1,\infty)$$ -For the [external tuning ruleset](#external-tuning-ruleset), we will only use $1$ study instead of the proposed $5$, when evaluating on the qualification set. The [self-tuning ruleset](#self-tuning-ruleset) will use $5$ studies on the qualification set as well since it is computationally cheaper. +This performance ratio $r_{s,w}$ expresses the "time spent by submission $s$ on workload $w$" relative to the "time spent by the best submission on this workload". E.g. If a submission takes twice as long on a particular workload compared to the best submission on this workload it will have a performance ratio of $2$. Lower performance ratios are therefore better, with an optimal ratio of $1$ if the given submission is the fastest on this workload. -### Scoring +##### Building performance profiles -Submissions will be scored based on their required training time to reach the target performance on the validation set of each workload. This target performance metric can be the same as the loss function but might also be a different workload-specific metric such as the error rate or BLEU score. The target performance was defined using four standard training algorithms, see the "[Defining target performance](#defining-target-performance)" section for more details. The training time of a submission includes the compilation times for computation graphs and ops that could happen just-in-time during training; all our benchmarks should be fast enough to compile so as not to dramatically impact overall performance. The overall ranking is then determined by summarizing the performances across all [fixed workloads](#fixed-workloads), using [performance profiles](#benchmark-score-using-performance-profiles), as explained below. +Next, we compute how often a submission is within a factor $\tau \in [1,\infty)$ of the optimal submission. For this, we determine the following function for every submission $\bar{s}$: -The training time until the target performance on the test set was reached is not used in the scoring procedure but might be used for additional analysis of the competition results. +$$ +\rho_{\bar{s}}(\tau) = \frac{1}{n} \cdot \left| \\{ w \text{ such that } r_{\bar{s},w}\leq \tau \\}\right| = \frac{1}{n} \cdot \left[\text{number of workloads where}\, r_{\bar{s},w}\leq \tau\right] +$$ -#### Benchmarking hardware +In other words, we compute the fraction of workloads where a submission $\bar{s}$ is less than $\tau$ away from the optimal submission. The function $\rho_{\bar{s}}(\tau)$ is monotonically increasing with $\tau$ and bounded between $0$ and $1$. -All scored runs have to be performed on the benchmarking hardware to allow for a fair comparison of training times. The benchmarking hardware has to be chosen to be easily accessible via common cloud computing providers. The exact hardware specification will most likely change with each iteration of the benchmark. The specs of the benchmarking hardware for this iteration of the benchmark are: +An example of a performance profiles plot is shown below, where we plot $\rho_{\bar{s}}(\tau)$ for six submissions: -- 8xV100 16GB GPUs -- 240 GB in RAM -- 2 TB in storage (for datasets). +![Example performance profile](/.assets/performance_profiles.png) -> [!NOTE] -> Submitters are no longer required to self-report results to enter the AlgoPerf competition. Instead, they can open a PR and the working group will score the most promising submissions, see our [How to Submit](/README.md#how-to-submit) section for more details. +##### Integrating performance profiles for the benchmark score -When self-reported results, it is acceptable to perform the tuning trials on hardware different from the benchmarking hardware, as long as the same hardware is used for all tuning trials. Once the best trial, i.e. the one that reached the *validation* target the fastest, was determined, this run has to be repeated on the competition hardware. For example, submitters can tune using their locally available hardware but have to use the benchmarking hardware, e.g. via cloud providers, for the $5$ scored runs. This allows for a fair comparison to the reported results of other submitters while allowing some flexibility in the hardware. +To get the scalar _AlgoPerf Benchmark Score_ that is usable for ranking submissions, we will integrate the performance profiles $\rho_{\bar{s}}(\tau)$ of all submissions to get their _AlgoPerf Benchmark Score_ $B_{\bar{s}}$, with -#### Defining target performance +$$B_{\bar{s}} = \frac{1}{r_{\text{max}}-1} \int_{1}^{r_{\text{max}}} \rho_{\bar{s}}(\tau) \,d\tau \in [0, 1].$$ -Target performances on the validation and test sets will be defined for each [workload](#workloads) separately. For the [fixed workloads](#fixed-workloads), we take the best performance achievable by one of four standard algorithms (AdamW, NadamW, Nesterov Momentum, and Heavy Ball Momentum). These target-setting algorithms will follow the general process of the external tuning ruleset, with a significantly larger tuning budget of $200$ trials to guarantee competitive performance. Once the best algorithm and its hyperparameters are determined, training is repeated $20$ times. The median of the best achieved validation errors across seeds is used as the *validation* target. Out of the $10$ repeated runs that achieved this validation target, we took the worst achieved test error across seeds as our *test* target. Taking the median validation performance after rerunning the best hyperparameter point prevents our procedure from selecting a lucky outlier. -To save computational resources, we only tuned two training algorithms instead of four, for the [randomized workloads](#randomized-workloads). For each workload variant, we used NadamW and the other best-performing training algorithm on the corresponding fixed workload the randomized workload is based on. +The upper integration limit will be set to $r_{\text{max}} = 4$ which also serves as the upper limit of the performance profile plot. +This means that any submission that requires more than four times the runtime of the fastest submission will not get any credit on this workload compared to a training algorithm that is unable to successfully train within the maximum allowed runtime budget. +The integral is normalized by the total integration area, such that all _AlgoPerf Benchmark scores_ are between $0$ and $1$, with higher scores being better. A perfect score of $1$ is achieved if a submission is the fastest on all workloads. -Both [tuning rulesets](#tuning) will use the same target performances. The runtime of the target-setting algorithms on each workload will be chosen to match published results and is constrained by the overall time budget of roughly a single week for all fixed workloads. The initial `max_runtime` for submissions on each workload was $\frac{1}{3}$ longer than the runtime of the target-setting algorithms (this `max_runtime` will be $1.5$ times as much for the self-tuning ruleset, see the [Self-tuning ruleset](#self-tuning-ruleset) section). After the initial round of submissions, we have adapated the `max_runtime` based on the performance of the submissions (see [this issue](https://github.com/mlcommons/algorithmic-efficiency/issues/836)). +##### Alternative scores -#### Benchmark score using performance profiles +Performance profiles and the _AlgoPerf Benchmark Score_ derived from them, take a bit of effort to explain. +However, we believe that they are fairer and well-supported by research in machine learning and the optimization community. To have some simpler to interpret numbers, e.g. for press releases, we will also release a series of alternative scores. -We will aggregate the training times of a submission on all fixed workloads using [Performance Profiles](http://www.argmin.net/2018/03/26/performance-profiles/) (originally from [Dolan and Moré](https://arxiv.org/abs/cs/0102001)). Below we surface several relevant definitions from their work for easier readability, before explaining how we integrate the performance profiles to reach a scalar benchmark score that will be used for ranking submissions. +For a given workload $\bar{w}$, we define the "speedup of a submission $\bar{s}$ over submission $\text{ref}$" as $\frac{t_{\text{ref}, \bar{w}}}{t_{\bar{s}, \bar{w}}}$. For example, if a submission was $2\times$ faster than the reference submission, this would be equal to $2$. In addition to the raw $t_{s,w}$ values, we will release the geometric mean of the speedups across all workloads, i.e. $\left(\prod_{w \in \mathcal{W}} \frac{t_{\text{ref}, w}}{t_{\bar{s}, w}}\right)^{\frac{1}{n}}$. -*Notation:* We have a set $\mathcal{S} = \{s_1, s_2, \dots, s_k\}$ of in total $k$ submissions that we evaluate on a set of $n$ fixed workloads: $\mathcal{W} = \{w_1, w_2, \dots, w_n\}$. For each submission $s$ and each workload $w$ we have a training time score $t_{s,w} \in [0,\infty)$. This is the time it took the submission to reach the validation target performance on this particular workload. +#### Benchmarking Hardware -##### Computing performance ratios +All officially scored runs will be performed on the same benchmarking hardware to allow for a fair comparison of wall-clock training times. +This benchmarking hardware is chosen to be easily accessible via common cloud computing providers and will likely change with each iteration of the benchmark. +The specs of the benchmarking hardware for this iteration of the benchmark are: -For all workloads and submissions, we first compute their performance ratio $r$, which is defined for a particular submission $\bar{s}$ and a particular workload $\bar{w}$ to be: +- 8× NVIDIA V100 (16 GB) GPUs +- 240 GB in RAM +- 2 TB in storage (for datasets). -$$r_{\bar{s},\bar{w}} = \frac{t_{\bar{s},\bar{w}}}{\min_{s \in \mathcal{S}} t_{s,\bar{w}}} \in [1,\infty)$$ +> [!TIP] +> Submitters are no longer required to self-report results to enter the AlgoPerf benchmark. +> Instead, they can open a PR and the working group will score the most promising submissions, see our [**How to Submit**](/README.md#how-to-submit) section for more details. +> If you'd like to self-report results, e.g., for paper experiments or to provide evidence of your submission's performance, it is possible to use a different hardware. However, we strongly recommend to use the same hardware for all algorithms, at least for the scored runs. It is possible to _perform tuning trials on different hardware_, as long as the hardware is consistent for all tuning trials. +> However, in order to compare to the published results, you will have to repeat at least those fastest trials on the benchmarking hardware. +> This allows for a fair comparison to the reported results of other submitters while allowing some flexibility in the hardware. -This performance ratio $r_{s,w}$ expresses the "time spent by submission $s$ on workload $w$" relative to the "time spent by the best submission on this workload". E.g. If a submission takes twice as long on a particular workload compared to the best submission on this workload it will have a performance ratio of $2$. Lower performance ratios are therefore better, with an optimal ratio of $1$ if the given submission is the fastest on this workload. +#### Defining Target Performance and `max_runtime` -##### Building performance profiles +This section briefly explains the process to define the target performance for each [**workload**](#workloads), which will be used by both [**tuning rulesets**](#tuning-rulesets) equally. For more details, see [**our benchmark paper**](https://arxiv.org/abs/2306.07179). -Next, we compute how often a submission is within a factor $\tau \in [1,\infty)$ of the optimal submission. For this, we determine the following function for every submission $\bar{s}$: +For each workload, we take the best performance achievable by one of four standard algorithms (`AdamW`, `NadamW`, `Nesterov Momentum`, and `Heavy Ball Momentum`). These target-setting algorithms will follow the general process of the external tuning ruleset, with a significantly larger tuning budget of $200$ trials to guarantee competitive performance. Once the best algorithm and its hyperparameters are determined, training is repeated $20$ times with this configuration. The median of the best achieved validation errors across seeds is used as the _validation_ target. Out of the $10$ repeated runs that achieved this validation target, we took the worst achieved test error across seeds as our _test_ target. Taking the median validation performance after rerunning the best hyperparameter point prevents our procedure from selecting a lucky outlier. -$$\rho_{\bar{s}}(\tau) = \left(\frac{1}{n}\right) \cdot \left[\text{number of workloads where}\, r_{\bar{s},w}\leq \tau\right]$$ +> [!NOTE] +> The runtime of the target-setting algorithms was chosen to roughly match published results without extending the overall benchmark budget too much. +> The initial `max_runtime` (used in version 0.5 of the benchmark) available to submissions on each workload was $\frac{1}{3}$ longer than the runtime of the target-setting algorithms to allow submissions a bit more time to reach the target on some workloads, if they can make up for it on others. After the initial competition, we have adapted the `max_runtimes` based on the performance of the submissions (see [this issue](https://github.com/mlcommons/algorithmic-efficiency/issues/836)). -In other words, we compute the fraction of workloads where a submission $\bar{s}$ is less than $\tau$ away from the optimal submission. The function $\rho_{\bar{s}}(\tau)$ is monotonically increasing with $\tau$ and bounded between $0$ and $1$. +## Versioning Policy -An example of a performance profiles plot is shown below, where we plot $\rho_{\bar{s}}(\tau)$ for seven "submissions": +_AlgoPerf_ uses a unified versioning scheme: codebase, rules, and leaderboard all share the same `Major.Minor` version. `Patch` versions may differ for minor, non-breaking updates to each component. All results produced under the same `Major.Minor` version are comparable, making it easy to cite "`AlgoPerf v0.X`" and know exactly which set of rules, code, and submissions are being referenced. -![Example performance profile](/.assets/performance_profiles.png) +- **Codebase:** The version is automatically set from the latest GitHub tag and accessible via `algoperf.__version__`. +- **Rules/Documentation:** This document reflects the unified version shown above. +- **Leaderboard:** The leaderboard in the [**submissions repository**](https://github.com/mlcommons/submissions_algorithms) displays which version of the benchmark was used for scoring. -##### Integrating performance profiles for the benchmark score +For detailed information about releases and version history, see our [**README**](../README.md#releases--roadmap) and our [**Changelog**](CHANGELOG.md). -To get a scalar score that is usable for ranking submissions, we will integrate the performance profiles $\rho_{\bar{s}}(\tau)$ of all submissions to get their benchmark score $B_{\bar{s}}$, with +### Version Freeze -$$B_{\bar{s}} = \frac{1}{r_{\text{max}}-1} \int_{1}^{r_{\text{max}}} \rho_{\bar{s}}(\tau) \,d\tau \in [0, 1].$$ +To ensure that all submitters can develop their submissions based on the same code that will be utilized for scoring, we freeze the package versions of the codebase dependencies in between benchmark versions. By doing so, we level the playing field for everyone involved, ensuring fairness and consistency in the assessment of submissions. We will try to minimize changes to the benchmark codebase as best as possible. -The upper integration limit will be set to $r_{\text{max}} = 4$ which also serves as the upper limit of the performance profile plot. -This means that any submission that requires more than four times the runtime of the fastest submission will not get any credit on this workload compared to a training algorithm that is unable to successfully train within the maximum allowed runtime budget. -The integral is normalized by the total integration area, with higher benchmark scores being better. +## License and Legal Requirements -##### Using held-out workloads in scoring +All submissions must be licensed under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0). +Furthermore, all submitters must sign the following agreements: -For the benchmark score, we compute and integrate the performance profiles using the training times of only the fixed workloads. But we use the submission's performance on the held-out workloads to penalize submissions. Specifically, if a submission is unable to train a held-out workload, we score the submission on the corresponding fixed workload as if that submission did not reach the target. In other words, for a submission to receive a finite training time on a fixed workload, it needs to: +- A signed [Contributor License Agreement (CLA) "Corporate CLA"](https://mlcommons.org/en/policies/) of MLCommons. +- _Either_ a membership in MLCommons _or_ a signed [non-member test agreement](https://mlcommons.org/en/policies/). +- A signed trademark license agreement, either the member or the non-member version, as appropriate. These license agreements are available upon request to [support@mlcommons.org](mailto:support@mlcommons.org). -1. Reach the validation target on the fixed workload within the maximum runtime. -2. Reach the validation target fixed workload within 4x of the fastest submission. -3. Reach the validation target on the held-out workload (corresponding to the fixed workload) within the maximum runtime. -4. Reach the validation target on the held-out workload (corresponding to the fixed workload) within 4x of the fastest submission. To determine the fastest submission on a held-out workload, we only consider submissions that reached the target on the corresponding fixed workload. This protects us against extremely fast submissions that only work on a specific held-out workload and are useless as general algorithms. +## FAQs -For fixed workloads without a corresponding held-out workload (e.g. in this iteration this is the case for one ImageNet and one LibriSpeech workload, since we only sample one held-out workload *per dataset*), requirements 3 and 4 are automatically satisfied. +> If your question isn't answered here, please [**contact us**](mailto:algorithms-chairs@mlcommons.org). These FAQs serve to supplement and clarify the rules and documentation described above. -Only if all four requirements are met, does the submission get a finite score. Otherwise, a submission will receive a training time of infinity. Note that the tuning process works the same for held-out workloads as for the fixed workloads, i.e. in the external tuning ruleset there are multiple tuning trials and only the fastest trial per study is relevant for scoring. +### Setup & Platform -This essentially means that being unable to successfully train a held-out workload can "disqualify" a submission from getting a good score on the fixed workload it is based on. In other words, we require submissions to be robust enough to handle workload variations. This protocol ensures that we prioritize the fixed workloads for scoring since they are the most relevant version of that workload in practice. However, we also protect our benchmark from egregious workload-specific tuning and penalize brittle methods that break with slight modifications of the workload. +
+My machine only has one GPU. How can I use this repo? -##### Alternative scores +> You can run this repo on a machine with an arbitrary number of GPUs. However, the default batch sizes of our algorithms collection (e.g. `algorithms/`) are tuned for a machine with 8× NVIDIA V100 (16 GB) GPUs. You may run into OOMs if you run these algorithms with fewer than 8 GPUs. If you run into these issues because you are using a machine with less total GPU memory, please reduce the batch sizes for the submission. Note that your final submission must 'fit' on the [**benchmarking hardware**](#benchmarking-hardware), so if you are using fewer GPUs with higher per-GPU memory, please monitor your memory usage to make sure it will fit on 8× NVIDIA V100 GPUs with 16 GB of VRAM per card. -Performance profiles and the benchmark score derived from them, take a bit of effort to explain. -However, we believe that they are fairer and well-supported by research in machine learning and the optimization community. To have some simpler to interpret numbers, e.g. for press releases, we will also release a series of alternative scores. +
-For a given workload $\bar{w}$, we define the "speedup of a submission $\bar{s}$ over the target-setting reference" as $\frac{t_{\text{ref}, \bar{w}}}{t_{\bar{s}, \bar{w}}}$. For example, if a submission was 2x faster than the target-setting reference, this would be equal to 2. In addition to the raw $t_{s,w}$ values, we will release the geometric mean of the speedups across all workloads, i.e. $\left(\prod_{w \in \mathcal{W}} \frac{t_{\text{ref}, w}}{t_{\bar{s}, w}}\right)^{\frac{1}{n}}$. +
+How do I run this on my SLURM cluster? -##### Verifying scores +> You may run into issues with `sudo` and `docker` on a SLURM cluster. To run the workloads in a SLURM cluster you can use Apptainer (_formerly Singularity_), see this [**section**](/docs/GETTING_STARTED.md#using-singularityapptainer-instead-of-docker). -The working group will independently verify the scores of the highest-scoring submissions in each ruleset. Results that have been verified by the working group will be clearly marked on the leaderboard. +
-### Version freeze +
+How can I run this on my AWS/GCP/Azure cloud project? -To ensure that all submitters can develop their submissions based on the same code that will be utilized for scoring, we freeze the package versions of the codebase dependencies in between benchmark iterations. By doing so, we level the playing field for everyone involved, ensuring fairness and consistency in the assessment of submissions. We will try to minimize changes to the benchmark codebase as best as possible. +> Depending on your virtual machine, you may have to install the correct GPU drivers and the NVIDIA Docker toolkit. For example, in GCP you will have to do the following. +> +> 1. If you don't have a VM instance yet, we recommend creating a +> new Compute Instance with the "Deep Learning on Linux" Image in Boot disk options. +> 2. To install the NVIDIA Docker toolkit, you can use [`docker/scripts/cloud-startup.sh`](/docker/scripts/cloud-startup.sh) as a startup script for the VM. This will automate the installation of the NVIDIA GPU Drivers and NVIDIA Docker toolkit. -## FAQs +
-### Setup and Platform +### Submitting -#### My machine only has one GPU. How can I use this repo? +
+How do I submit my algorithm to the benchmark? -You can run this repo on a machine with an arbitrary number of GPUs. However, the default batch sizes in our reference algorithms (e.g. `algorithmic-efficiency/prize_qualification_baselines` and `algorithmic-efficiency/reference_algorithms`) are tuned for a machine with 8 16GB V100 GPUs. You may run into OOMs if you run these algorithms with fewer than 8 GPUs. If you run into these issues because you are using a machine with less total GPU memory, please reduce the batch sizes for the submission. Note that your final submission must 'fit' on the benchmarking hardware, so if you are using fewer -GPUs with higher per GPU memory, please monitor your memory usage to make sure it will fit on 8xV100 GPUs with 16GB of VRAM per card. +> Please see our [**How to Submit**](/README.md#how-to-submit) section. You can submit your algorithm to the benchmark by opening a PR on the [**submission repository**](https://github.com/mlcommons/submissions_algorithms). -#### How do I run this on my SLURM cluster? +
-You may run into issues with `sudo` and `docker` on a SLURM cluster. To run the workloads in a SLURM cluster you can use Apptainer (previously Singularity), see this [section](/GETTING_STARTED.md#using-singularityapptainer-instead-of-docker). +
+Can I submit multiple times? -#### How can I run this on my AWS/GCP/Azure cloud project? +> Our benchmark allows multiple submissions as long as they are substantially different. We discourage submitters from creating bulk submissions as this is not in the spirit of the benchmark. -Depending on your virtual machine, you may have to install the correct GPU drivers and the NVIDIA Docker toolkit. For example, in GCP you will have to do the following. +
-1. If you don't have a VM instance yet, we recommend creating a -new Compute Instance with the "Deep Learning on Linux" Image in Boot disk options. -2. To install the NVIDIA Docker toolkit, you can use `scripts/cloud-startup.sh` as a startup script for the VM. This will automate the installation of the NVIDIA GPU Drivers and NVIDIA Docker toolkit. +
+Can my submission span multiple files? -### Submitting +> Yes, your submission can be structured using multiple files. -#### How do I submit my algorithm to the benchmark? +
-Please see our [How to Submit](/README.md#how-to-submit) section. You can submit your algorithm to the benchmark by opening a PR on the [submission repository](https://github.com/mlcommons/submissions_algorithms). +
+Can I install custom dependencies? -#### Can I submit multiple times to the benchmark competition? +> You may use custom dependencies as long as they do not conflict with any of the pinned packages in [`pyproject.toml`](/pyproject.toml). +> +> To include your custom dependencies in your submission, please include them in a `requirements.txt` file. Please refer to the [**Software dependencies**](#software-dependencies) section of our rules. -Our benchmark allows multiple submissions by the same team of submitters as long as they are substantially different. We discourage submitters from creating bulk submissions as this is not in the spirit of the benchmark. +
-#### Can my submission be structured using multiple files? +### Scoring & Hardware -Yes, your submission can be structured using multiple files. +
+How can I know if my code can be run on benchmarking hardware? -#### Can I install custom dependencies? +> The benchmarking hardware specifications are documented in the [**Benchmarking Hardware Section**](#benchmarking-hardware). We recommend monitoring your submission's memory usage so that it does not exceed the available memory on the benchmarking hardware. We also recommend doing a dry run using a cloud instance. -You may use custom dependencies as long as they do not conflict with any of the pinned packages in `algorithmic-efficiency/setup.cfg`. -To include your custom dependencies in your submission, please include them in a requirements.txt file. Please refer to the [Software dependencies](#software-dependencies) section of our rules. +
-#### How can I know if my code can be run on benchmarking hardware? +
+This benchmark seems computationally expensive. Do I have to run it myself? -The benchmarking hardware specifications are documented in the [Benchmarking Hardware Section](#benchmarking-hardware). We recommend monitoring your submission's memory usage so that it does not exceed the available memory -on the benchmarking hardware. We also recommend to do a dry run using a cloud instance. +> Submitters are no longer required to self-report results to get on the _AlgoPerf_ leaderboard. Instead, they can open a PR in the [**submission repository**](https://github.com/mlcommons/submissions_algorithms) and the working group will score the most promising submissions, see our [**How to Submit**](/README.md#how-to-submit) section for more details. You can use self-reported results to provide evidence of performance on the benchmark. Even if you fully self-report, we will still verify the scores by rerunning the submission on our setup. -#### This benchmark seems computationally expensive. Do I have to run it myself? +
-Submitters are no longer required to self-report results to get on the AlgoPerf leaderboard. Instead, they can open a PR and the working group will score the most promising submissions, see our [How to Submit](/README.md#how-to-submit) section for more details. You can use self-reported results to provide evidence of performance on the benchmark. Even if you fully self-report, we will still verify the scores by rerunning the submission on our setup. +
+Can I submit previously published training algorithms as submissions? -#### Can I submit previously published training algorithms as submissions? +> Yes, you may, as long as it isn't an exact copy of an existing submission. +> +> For example, you may submit the Adam optimizer with your particularly effective hyperparameter search space and hyperparameter configuration, as different choices for hyperparameter values and/or search spaces constitute different training algorithms and are potential sources of innovation. +> +> That said, while submitting Adam with some novel heuristic to set various hyperparameters, some especially effective hyperparameter search space, or your single best hyperparameter configuration is fine, avoid making multiple submissions that only differ by their hyperparameter configuration without a convincing justification they are substantially different (see the [**"Can I submit multiple times to the benchmark competition?"**](#submitting) question, above). -Yes, you may, as long as it isn't an exact copy of an existing submission. -For example, you may submit the Adam optimizer with your particularly effective hyperparameter search space and hyperparameter configuration, as different choices for hyperparameter values and/or search spaces constitute different training algorithms and are potential sources of innovation. -That said, while submitting Adam with some novel heuristic to set various hyperparameters, some especially effective hyperparameter search space, or your single best hyperparameter configuration is fine, avoid making multiple submissions that only differ by their hyperparameter configuration without a convincing justification they are substantially different (see ["Can I submit multiple times to the benchmark competition?"](/COMPETITION_RULES.md#can-i-submit-multiple-times-to-the-benchmark-competition), above). +
## Disclaimers -### Shared Data Pipelines between JAX and PyTorch +### Shared Data Pipelines between `JAX` and `PyTorch` -The JAX and PyTorch versions of the Criteo, FastMRI, Librispeech, OGBG, and WMT workloads use the same TensorFlow input pipelines. Due to differences in how JAX and PyTorch distribute computations across devices, the PyTorch workloads have an additional overhead for these workloads. +The `JAX` and `PyTorch` versions of the `Criteo`, `fastMRI`, `LibriSpeech`, `OGBG`, and `WMT` workloads use the same `TensorFlow` input pipelines. Due to differences in how `JAX` and `PyTorch` distribute computations across devices, the `PyTorch` workloads have an additional overhead for these workloads. -Since we use PyTorch's [`DistributedDataParallel`](https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html#torch.nn.parallel.DistributedDataParallel) implementation, there is one Python process for each device. Depending on the hardware and the settings of the cluster, running a TensorFlow input pipeline in each Python process can lead to errors, since too many threads are created in each process. See [this PR thread](https://github.com/mlcommons/algorithmic-efficiency/pull/85) for more details. -While this issue might not affect all setups, we currently implement a different strategy: we only run the TensorFlow input pipeline in one Python process (with `rank == 0`), and [broadcast](https://pytorch.org/docs/stable/distributed.html#torch.distributed.broadcast) the batches to all other devices. This introduces an additional communication overhead for each batch. See the [implementation for the WMT workload](https://github.com/mlcommons/algorithmic-efficiency/blob/main/algoperf/workloads/wmt/wmt_pytorch/workload.py#L215-L288) as an example. +Since we use `PyTorch`'s [`DistributedDataParallel`](https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html#torch.nn.parallel.DistributedDataParallel) implementation, there is one Python process for each device. Depending on the hardware and the settings of the cluster, running a `TensorFlow` input pipeline in each Python process can lead to errors, since too many threads are created in each process. See [this PR thread](https://github.com/mlcommons/algorithmic-efficiency/pull/85) for more details. +While this issue might not affect all setups, we currently implement a different strategy: we only run the `TensorFlow` input pipeline in one Python process (with `rank == 0`), and [broadcast](https://pytorch.org/docs/stable/distributed.html#torch.distributed.broadcast) the batches to all other devices. This introduces additional communication overhead for each batch. See the [implementation for the `WMT` workload](https://github.com/mlcommons/algorithmic-efficiency/blob/main/algoperf/workloads/wmt/wmt_pytorch/workload.py#L215-L288) as an example. diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index d2995ba49..0cc286099 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -26,10 +26,10 @@ To get started you will have to make a few decisions and install the repository 1. Decide if you would like to develop your submission in either PyTorch or JAX. 2. Set up your workstation or VM. We recommend to use a setup similar to the [benchmarking hardware](/DOCUMENTATION.md#benchmarking-hardware). -The specs on the benchmarking machines are: - - 8xV100 16GB GPUs - - 240 GB in RAM - - 2 TB in storage (for datasets). + The specs on the benchmarking machines are: + - 8xV100 16GB GPUs + - 240 GB in RAM + - 2 TB in storage (for datasets). 3. Install the `algoperf` package and dependencies either in a [Python virtual environment](#python-virtual-environment) or use a [Docker](#docker) (recommended) or [Singularity/Apptainer container](#using-singularityapptainer-instead-of-docker). ### Python Virtual Environment @@ -44,42 +44,42 @@ To set up a virtual enviornment and install this repository 1. Create new environment, e.g. via `conda` or `virtualenv` - ```bash - sudo apt-get install python3-venv - python3 -m venv env - source env/bin/activate - ``` + ```bash + sudo apt-get install python3-venv + python3 -m venv env + source env/bin/activate + ``` 2. Clone this repository - ```bash - git clone https://github.com/mlcommons/algorithmic-efficiency.git - cd algorithmic-efficiency - ``` + ```bash + git clone https://github.com/mlcommons/algorithmic-efficiency.git + cd algorithmic-efficiency + ``` 3. Run the following pip3 install commands based on your chosen framework to install `algoperf` and its dependencies. - For **JAX**: + For **JAX**: - ```bash - pip3 install -e '.[pytorch_cpu]' - pip3 install -e '.[jax_gpu]' -f 'https://storage.googleapis.com/jax-releases/jax_cuda_releases.html' - pip3 install -e '.[full]' - ``` + ```bash + pip3 install -e '.[pytorch_cpu]' + pip3 install -e '.[jax_gpu]' -f 'https://storage.googleapis.com/jax-releases/jax_cuda_releases.html' + pip3 install -e '.[full]' + ``` - For **PyTorch** + For **PyTorch** - Note: the below command assumes you have CUDA 12.1 installed locally. - This is the default in the provided Docker image. - We recommend you match this CUDA version but if you decide to run - with a different local CUDA version, please find the appropriate wheel - url to pass to the `pip install` command for `pytorch`. + Note: the below command assumes you have CUDA 12.1 installed locally. + This is the default in the provided Docker image. + We recommend you match this CUDA version but if you decide to run + with a different local CUDA version, please find the appropriate wheel + url to pass to the `pip install` command for `pytorch`. - ```bash - pip3 install -e '.[jax_cpu]' - pip3 install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/cu121' - pip3 install -e '.[full]' - ``` + ```bash + pip3 install -e '.[jax_cpu]' + pip3 install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/cu121' + pip3 install -e '.[full]' + ```
@@ -106,7 +106,7 @@ We recommend using a Docker container to ensure a similar environment to our sco > **Prerequisites:** > > - NVIDIA Driver version 535.104.05 -> - NVIDIA Container Toolkit so that the containers can locate the NVIDIA drivers and GPUs. See instructions [here](https://github.com/NVIDIA/nvidia-docker). +> - NVIDIA Container Toolkit so that the containers can locate the NVIDIA drivers and GPUs. See instructions in the [NVIDIA Docker documentation](https://github.com/NVIDIA/nvidia-docker). #### Building Docker Image @@ -132,29 +132,29 @@ To use the Docker container as an interactive virtual environment, you can run a 1. Run detached Docker container. The `container_id` will be printed if the container is run successfully. - ```bash - docker run -t -d \ - -v $HOME/data/:/data/ \ - -v $HOME/experiment_runs/:/experiment_runs \ - -v $HOME/experiment_runs/logs:/logs \ - -v $HOME/algorithmic-efficiency:/algorithmic-efficiency \ - --gpus all \ - --ipc=host \ - \ - --keep_container_alive true - ``` - - > Note: You may have to use double quotes around `algorithmic-efficiency` [path] in the mounting `-v` flag. If the above command fails try replacing the following line: - > - > ```bash - > -v $HOME/algorithmic-efficiency:/algorithmic-efficiency2 \ - > ``` - > - > with - > - > ```bash - > -v $HOME"/algorithmic-efficiency:/algorithmic-efficiency" \ - > ``` + ```bash + docker run -t -d \ + -v $HOME/data/:/data/ \ + -v $HOME/experiment_runs/:/experiment_runs \ + -v $HOME/experiment_runs/logs:/logs \ + -v $HOME/algorithmic-efficiency:/algorithmic-efficiency \ + --gpus all \ + --ipc=host \ + \ + --keep_container_alive true + ``` + + > Note: You may have to use double quotes around `algorithmic-efficiency` [path] in the mounting `-v` flag. If the above command fails try replacing the following line: + > + > ```bash + > -v $HOME/algorithmic-efficiency:/algorithmic-efficiency2 \ + > ``` + > + > with + > + > ```bash + > -v $HOME"/algorithmic-efficiency:/algorithmic-efficiency" \ + > ``` 2. Open a bash terminal @@ -177,7 +177,7 @@ singularity shell --bind $HOME/data:/data,$HOME/experiment_runs:/experiment_runs --nv .sif ``` -Note the `--bind` flag which, similarly to Docker, allows to bind specific paths on the host system and the container, as explained [here](https://docs.sylabs.io/guides/3.7/user-guide/bind_paths_and_mounts.html). +Note the `--bind` flag which, similarly to Docker, allows to bind specific paths on the host system and the container, as explained in the [Singularity User Guide](https://docs.sylabs.io/guides/3.7/user-guide/bind_paths_and_mounts.html). Also note that we generated `Singularity.def` automatically from the `Dockerfile` using [spython](https://github.com/singularityhub/singularity-cli), as follows: @@ -203,41 +203,42 @@ Make a submissions subdirectory to store your submission modules e.g. `algorithm ### Coding your Submission -You can find examples of submission modules under `algorithmic-efficiency/prize_qualification_baselines` and `algorithmic-efficiency/reference_algorithms`. \ +You can find examples of submission modules under `algorithmic-efficiency/algorithms`. \ A submission for the external ruleset will consist of a submission module and a tuning search space definition. -1. Copy the template submission module `submissions/template/submission.py` into your submissions directory e.g. in `algorithmic-efficiency/my_submissions`. +1. Copy the template submission module `algorithms/template/submission.py` into your submissions directory e.g. in `algorithmic-efficiency/my_submissions`. 2. Implement at least the methods in the template submission module. Feel free to use helper functions and/or modules as you see fit. Make sure you adhere to to the competition rules. Check out the guidelines for [allowed submissions](/DOCUMENTATION.md#allowed-submissions), [disallowed submissions](/DOCUMENTATION.md#allowed-submissions) and pay special attention to the [software dependencies rule](/DOCUMENTATION.md#software-dependencies). 3. Add a tuning configuration e.g. `tuning_search_space.json` file to your submission directory. For the tuning search space you can either: - 1. Define the set of feasible points by defining a value for "feasible_points" for the hyperparameters: - ```JSON - { - "learning_rate": { - "feasible_points": 0.999 - }, - } - ``` + 1. Define the set of feasible points by defining a value for "feasible_points" for the hyperparameters: - For a complete example see [tuning_search_space.json](/reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json). + ```JSON + { + "learning_rate": { + "feasible_points": 0.999 + }, + } + ``` - 2. Define a range of values for quasirandom sampling by specifing a `min`, `max` and `scaling` keys for the hyperparameter: + For a complete example see [tuning_search_space.json](/algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json). - ```JSON - { - "weight_decay": { - "min": 5e-3, - "max": 1.0, - "scaling": "log", - } - } - ``` + 2. Define a range of values for quasirandom sampling by specifing a `min`, `max` and `scaling` keys for the hyperparameter: - For a complete example see [tuning_search_space.json](/reference_algorithms/paper_baselines/nadamw/tuning_search_space.json). + ```JSON + { + "weight_decay": { + "min": 5e-3, + "max": 1.0, + "scaling": "log", + } + } + ``` + + For a complete example see [tuning_search_space.json](/algorithms/archived_paper_baselines/nadamw/tuning_search_space.json). ## Run your Submission -From your virtual environment or interactively running Docker container run your submission with `submission_runner.py`: +From your virtual environment or interactively running Docker container run your submission with `submission_runner.py`: **JAX**: to score your submission on a workload, from the algorithmic-efficency directory run: @@ -328,13 +329,13 @@ This will print the container ID to the terminal. To find the container IDs of running containers ```bash -docker ps +docker ps ``` To see output of the entrypoint script ```bash -docker logs +docker logs ``` To enter a bash session in the container @@ -345,24 +346,14 @@ docker exec -it /bin/bash ## Score your Submission -To score your submission we will score over all fixed workloads, held-out workloads and studies as described in the rules. -We will sample 1 held-out workload per dataset for a total of 6 held-out workloads and will use the sampled held-out workloads in the scoring criteria for the matching fixed base workloads. +To score your submission we will score over all workloads, studies, and trials as described in the rules. In other words, the total number of runs expected for official scoring is: -- for external tuning ruleset: **350** = (8 (fixed workloads) + 6 (held-out workloads)) x 5 (studies) x 5 (trials) -- for self-tuning ruleset: **70** = (8 (fixed workloads) + 6 (held-out workloads)) x 5 (studies) +- for external tuning ruleset: **120** = 8 (workloads) x 3 (studies) x 5 (trials) +- for self-tuning ruleset: **24** = 8 (workloads) x 3 (studies) ### Running workloads -To run workloads for (a mock) scoring you may specify a "virtual" list of held-out workloads. It is important to note that the official set of held-out workloads will be sampled by the competition organizers during scoring time. - -An example config for held-out workloads is stored in `scoring/held_workloads_example.json`. -To generate a new sample of held out workloads run: - -```bash -python3 generate_held_out_workloads.py --seed --output_filename -``` - To run a number of studies and trials over all workload using Docker containers for each run: ```bash @@ -379,14 +370,14 @@ python scoring/run_workloads.py \ Note that to run the above script you will need at least the `jax_cpu` and `pytorch_cpu` installations of the `algorithmic-efficiency` package. -During submission development, it might be useful to do faster, approximate scoring (e.g. without `5` different studies or when some trials are missing) so the scoring scripts allow some flexibility. +During submission development, it might be useful to do faster, approximate scoring (e.g. without `3` different studies or when some trials are missing) so the scoring scripts allow some flexibility. To simulate official scoring, pass the `--strict=True` flag in `score_submission.py`. To get the raw scores and performance profiles of group of submissions or single submission: ```bash python score_submissions.py --submission_directory --output_dir --compute_performance_profiles ``` -We provide the scores and performance profiles for the [paper baseline algorithms](/reference_algorithms/paper_baselines/) in the "Baseline Results" section in [Benchmarking Neural Network Training Algorithms](https://arxiv.org/abs/2306.07179). +We provide the scores and performance profiles for the [paper baseline algorithms](/algorithms/archived_paper_baselines/) in the "Baseline Results" section in [Benchmarking Neural Network Training Algorithms](https://arxiv.org/abs/2306.07179). ## Submit your Submission diff --git a/pyproject.toml b/pyproject.toml index b7afea8b5..e4de98f89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ zip-safe = false find = {} # Scanning implicit namespaces is active by default [tool.setuptools_scm] +# Version is automatically managed by setuptools_scm from GitHub tags. version_file = "algoperf/_version.py" ############################################################################### @@ -183,4 +184,4 @@ ignore = [ "FBT001", "FBT003", "TD003", -] \ No newline at end of file +] diff --git a/reference_algorithms/paper_baselines/README.md b/reference_algorithms/paper_baselines/README.md deleted file mode 100644 index 6c7027adf..000000000 --- a/reference_algorithms/paper_baselines/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Baseline Submissions from the "Benchmarking Neural Network Training Algorithms" Paper - -This directory contains baseline submissions for the [external tuning ruleset](../README.md#external-tuning-ruleset) as presented in our paper [Benchmarking Neural Network Training Algorithms](https://arxiv.org/abs/2306.07179): - -- [AdamW](/reference_algorithms/paper_baselines/adamw) -- [SGD with Momentum](/reference_algorithms/paper_baselines/momentum) -- [NadamW](/reference_algorithms/paper_baselines/nadamw) -- [SGD with Nesterov Momentum](/reference_algorithms/paper_baselines/nesterov) - -Each update rule has two different tuning search spaces, one where the first momentum parameter (often denoted $\beta_1$) is tuned and one where it is set to a fixed value. diff --git a/reference_algorithms/qualification_baselines/README.md b/reference_algorithms/qualification_baselines/README.md deleted file mode 100644 index 0281b8851..000000000 --- a/reference_algorithms/qualification_baselines/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Prize Qualification Baselines - -This directory contains the baseline(s) that submissions must beat to qualify for prizes, see the [Scoring Section](/COMPETITION_RULES.md#scoring) of the competition rules. For each ruleset there are 2 baselines (`*_target_setting.py` and `*_full_budget.py`). A submission must beat both baselines to be eligible for prizes. - -The experiment logs with training metrics are in `prize_qualification_baselines/logs` - -## Externally Tuned Ruleset - -### JAX - -The prize qualification baseline submissions for JAX are: - -- `prize_qualification_baselines/external_tuning/jax_nadamw_target_setting.py` -- `prize_qualification_baselines/external_tuning/jax_nadamw_full_budget.py` - -Example command: - -```bash -python3 submission_runner.py \ - --framework=jax \ - --data_dir= \ - --experiment_dir= \ - --experiment_name= \ - --workload= \ - --submission_path=prize_qualification_baselines/external_tuning/jax_nadamw_target_setting.py \ - --tuning_search_space=prize_qualification_baselines/external_tuning/tuning_search_space.json -``` - -### PyTorch - -The prize qualification baseline submissionss for PyTorch are: - -- `prize_qualification_baselines/external_tuning/pytorch_nadamw_target_setting.py` -- `prize_qualification_baselines/external_tuning/pytorch_nadamw_full_budget.py` - -Example command: - -```bash -torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ - --framework=pytorch \ - --data_dir= \ - --experiment_dir= \ - --experiment_name=t \ - --workload=\ - --submission_path=prize_qualification_baselines/external_tuning/pytorch_nadamw_target_setting.py \ - --tuning_search_space=prize_qualification_baselines/external_tuning/tuning_search_space.json -``` - -## Self-tuning Ruleset - -### JAX - -The prize qualification baseline submissionss for jax are: - -- `prize_qualification_baselines/self_tuning/jax_nadamw_target_setting.py` -- `prize_qualification_baselines/self_tuning/jax_nadamw_full_budget.py` - -Example command: - -```bash -python3 submission_runner.py \ - --framework=jax \ - --data_dir= \ - --experiment_dir= \ - --experiment_name= \ - --workload= \ - --submission_path=prize_qualification_baselines/self_tuning/jax_nadamw_target_setting.py \ - --tuning_ruleset=self -``` - -### PyTorch - -The prize qualification baseline submissionss for PyTorch are: - -- `prize_qualification_baselines/self_tuning/pytorch_nadamw_target_setting.py` -- `prize_qualification_baselines/self_tuning/pytorch_nadamw_full_budget.py` - -Example command: - -```bash -torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \ - --framework=pytorch \ - --data_dir= \ - --experiment_dir= \ - --experiment_name=t \ - --workload=\ - --submission_path=prize_qualification_baselines/self_tuning/pytorch_nadamw_target_setting.py \ - --tuning_ruleset=self -``` diff --git a/reference_algorithms/qualification_baselines/external_tuning/jax_nadamw_target_setting.py b/reference_algorithms/qualification_baselines/external_tuning/jax_nadamw_target_setting.py deleted file mode 100644 index d9088bf53..000000000 --- a/reference_algorithms/qualification_baselines/external_tuning/jax_nadamw_target_setting.py +++ /dev/null @@ -1,426 +0,0 @@ -"""Submission file for an NAdamW optimizer with warmup+cosine LR in Jax.""" - -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - NamedTuple, - Optional, - Tuple, - Union, -) - -import chex -import jax -import jax.numpy as jnp -import optax - -from algoperf import jax_sharding_utils, spec - -_GRAD_CLIP_EPS = 1e-6 - - -# Forked from -# github.com/google/init2winit/blob/master/init2winit/optimizer_lib/alias.py -def nadamw( - learning_rate: Union[float, optax.Schedule], - b1: float = 0.9, - b2: float = 0.999, - eps: float = 1e-8, - eps_root: float = 0.0, - debias: bool = True, - weight_decay: float = 0.0, - weight_decay_mask: Optional[Union[Any, Callable[[optax.Params], Any]]] = None, -) -> optax.GradientTransformation: - """Rescale updates according to the NAdam algorithm. - - References: - There seem to be multiple versions of NAdam. The original version is here - https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ (the official PyTorch - implementation also follows this). - Current code implements a simpler version with no momentum decay and slightly - different bias correction terms. The exact description can be found here - https://arxiv.org/pdf/1910.05446.pdf (Table 1). - - Args: - learning_rate: A fixed global scaling factor. - b1: Decay rate for the exponentially weighted average of grads. - b2: Decay rate for the exponentially weighted average of squared grads. - eps: Term added to the denominator to improve numerical stability. - eps_root: Term added to the denominator inside the square-root to improve - numerical stability when backpropagating gradients through the rescaling. - debias: Whether to use bias correction. - weight_decay: Strength of the weight decay regularization. Note that this - weight decay is multiplied with the learning rate. This is consistent with - other frameworks such as PyTorch, but different from (Loshchilov et al, - 2019) where the weight decay is only multiplied with the "schedule - multiplier", but not the base learning rate. - weight_decay_mask: A tree with same structure as (or a prefix of) the params - PyTree, or a Callable that returns such a pytree given the params/updates. - The leaves should be booleans, `True` for leaves/subtrees you want to - apply the weight decay to, and `False` for those you want to skip. Note - that the Nadam gradient transformations are applied to all parameters. - - Returns: - An (init_fn, update_fn) tuple. - """ - return optax.chain( - scale_by_nadam(b1, b2, eps, eps_root, debias), - optax.add_decayed_weights(weight_decay, weight_decay_mask), - scale_by_learning_rate(learning_rate), - ) - - -# All functions below are forked from -# github.com/google/init2winit/blob/master/init2winit/optimizer_lib/transform.py -def scale_by_nadam( - b1: float = 0.9, - b2: float = 0.999, - eps: float = 1e-8, - eps_root: float = 0.0, - debias: bool = True, - power: float = 0.5, -) -> optax.GradientTransformation: - """Rescale updates according to the NAdam algorithm. - - References: - There seem to be multiple versions of NAdam. The original version is here - https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ (the pytorch imp. also - follows this). - - Current code implements a simpler version with no momentum decay and slightly - different (standard Adam) bias correction terms. The exact description can be - found here https://arxiv.org/pdf/1910.05446.pdf (Table 1) - - Args: - b1: Decay rate for the exponentially weighted average of grads. - b2: Decay rate for the exponentially weighted average of squared grads. - eps: Term added to the denominator to improve numerical stability. - eps_root: Term added to the denominator inside the square-root to improve - numerical stability when backpropagating gradients through the rescaling. - debias: Whether to use bias correction. - power: The power to use in the preconditioner (0.5 in default adam). - Returns: - An (init_fn, update_fn) tuple. - """ - raise_power = jnp.sqrt if power == 0.5 else lambda x: jnp.power(x, power) - - def init_fn(params): - mu = jax.tree.map(jnp.zeros_like, params) # First moment - nu = jax.tree.map(jnp.zeros_like, params) # Second moment - return ScaleByAdamState(count=jnp.zeros([], jnp.int32), mu=mu, nu=nu) - - def update_fn(updates, state, params=None): - del params - mu = _update_moment(updates, state.mu, b1, 1) - nu = _update_moment(updates, state.nu, b2, 2) - count = state.count + jnp.array(1, dtype=jnp.int32) - mu_hat = _update_moment(updates, mu, b1, 1) - mu_hat = mu_hat if not debias else _bias_correction(mu_hat, b1, count) - nu_hat = nu if not debias else _bias_correction(nu, b2, count) - updates = jax.tree.map( - lambda m, v: m / (raise_power(v + eps_root) + eps), mu_hat, nu_hat - ) - return updates, ScaleByAdamState(count=count, mu=mu, nu=nu) - - return optax.GradientTransformation(init_fn, update_fn) - - -class ScaleByAdamState(NamedTuple): - """State for the NAdam algorithm.""" - - count: chex.Array # shape=(), dtype=jnp.int32. - mu: optax.Updates - nu: optax.Updates - - -def _update_moment(updates, moments, decay, order): - """Compute the exponential moving average of the `order-th` moment.""" - return jax.tree.map( - lambda g, t: (1 - decay) * (g**order) + decay * t, updates, moments - ) - - -def _bias_correction(moment, decay, count): - """Perform bias correction. This becomes a no-op as count goes to infinity.""" - beta = 1 - decay**count - return jax.tree.map(lambda t: t / beta.astype(t.dtype), moment) - - -def scale_by_learning_rate(learning_rate, flip_sign=True): - m = -1 if flip_sign else 1 - if callable(learning_rate): - return optax.scale_by_schedule(lambda count: m * learning_rate(count)) - return optax.scale(m * learning_rate) - - -def init_optimizer_state( - workload: spec.Workload, - model_params: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - rng: spec.RandomState, -) -> spec.OptimizerState: - """Creates a NAdamW optimizer and a learning rate schedule.""" - del model_params - del model_state - del rng - - def jax_cosine_warmup(step_hint: int, hyperparameters): - # Create learning rate schedule. - warmup_steps = int(hyperparameters.warmup_factor * step_hint) - warmup_fn = optax.linear_schedule( - init_value=0.0, - end_value=hyperparameters.learning_rate, - transition_steps=warmup_steps, - ) - cosine_steps = max(step_hint - warmup_steps, 1) - cosine_fn = optax.cosine_decay_schedule( - init_value=hyperparameters.learning_rate, decay_steps=cosine_steps - ) - schedule_fn = optax.join_schedules( - schedules=[warmup_fn, cosine_fn], boundaries=[warmup_steps] - ) - return schedule_fn - - # Create optimizer + LR schedule. - lr_schedule_fn = jax_cosine_warmup(workload.step_hint * 0.75, hyperparameters) - opt_init_fn, opt_update_fn = nadamw( - learning_rate=lr_schedule_fn, - b1=1.0 - hyperparameters.one_minus_beta1, - b2=hyperparameters.beta2, - eps=1e-8, - weight_decay=hyperparameters.weight_decay, - ) - params_zeros_like = jax.tree.map( - lambda s: jnp.zeros(s.shape_tuple), workload.param_shapes - ) - optimizer_state = opt_init_fn(params_zeros_like) - - return optimizer_state, opt_update_fn - - -def train_step( - workload, - opt_update_fn, - model_state, - optimizer_state, - current_param_container, - batch, - rng, - grad_clip, - label_smoothing, - dropout_rate, -): - def _loss_fn(params): - """Loss function used for training.""" - logits, new_model_state = workload.model_fn( - params, - batch, - model_state, - spec.ForwardPassMode.TRAIN, - rng, - update_batch_norm=True, - dropout_rate=dropout_rate, - ) - loss_dict = workload.loss_fn( - label_batch=batch['targets'], - logits_batch=logits, - mask_batch=batch.get('weights'), - label_smoothing=label_smoothing, - ) - summed_loss = loss_dict['summed'] - n_valid_examples = loss_dict['n_valid_examples'] - return summed_loss, (n_valid_examples, new_model_state) - - grad_fn = jax.value_and_grad(_loss_fn, has_aux=True) - (summed_loss, (n_valid_examples, new_model_state)), grad = grad_fn( - current_param_container - ) - # Get mean loss and grad. - loss = summed_loss / n_valid_examples - grad = jax.tree.map(lambda x: x / n_valid_examples, grad) - - grad_norm = jnp.sqrt( - sum(jnp.sum(g**2) for g in jax.tree_util.tree_leaves(grad)) - ) - - if grad_clip is not None: - grad_scaling_factor = grad_clip / (grad_norm + _GRAD_CLIP_EPS) - grad_scaling_factor = jax.lax.clamp(min=0.0, x=grad_scaling_factor, max=1.0) - grad = jax.tree.map(lambda x: x * grad_scaling_factor, grad) - - updates, new_optimizer_state = opt_update_fn( - grad, optimizer_state, current_param_container - ) - updated_params = optax.apply_updates(current_param_container, updates) - return new_optimizer_state, updated_params, new_model_state, loss, grad_norm - - -def update_params( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - batch: Dict[str, spec.Tensor], - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, - train_state: Optional[Dict[str, Any]] = None, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params, updated_model_state).""" - del current_params_types - del loss_type - del train_state - del eval_results - - optimizer_state, opt_update_fn = optimizer_state - if hasattr(hyperparameters, 'label_smoothing'): - label_smoothing = hyperparameters.label_smoothing - else: - label_smoothing = 0.0 - if hasattr(hyperparameters, 'grad_clip'): - grad_clip = hyperparameters.grad_clip - else: - grad_clip = None - dropout_rate = hyperparameters.dropout_rate - - # Create shardings for each argument - replicated = jax_sharding_utils.get_replicate_sharding() # No partitioning - sharded = ( - jax_sharding_utils.get_batch_dim_sharding() - ) # Partition along batch dimension - - # Create the sharding rules for each argument - arg_shardings = ( - # workload is static - # opt_update_fn is static - replicated, # model_state - replicated, # optimizer_state - replicated, # current_param_container - sharded, # batch - replicated, # rng - replicated, # grad_clip - replicated, # label_smoothing - replicated, # dropout_rate - ) - out_shardings = ( - replicated, # new_optimizer_state - replicated, # updated_params - replicated, # new_model_state - replicated, # loss - replicated, # grad_norm - ) - - # Jit with shardings - jitted_train_step = jax.jit( - train_step, - static_argnums=(0, 1), - donate_argnums=(2, 3, 4), - in_shardings=arg_shardings, - out_shardings=out_shardings, - ) - outputs = jitted_train_step( - workload, - opt_update_fn, - model_state, - optimizer_state, - current_param_container, - batch, - rng, - grad_clip, - label_smoothing, - dropout_rate, - ) - new_optimizer_state, new_params, new_model_state, loss, grad_norm = outputs - - # Log loss, grad_norm. - if global_step % 100 == 0 and workload.metrics_logger is not None: - workload.metrics_logger.append_scalar_metrics( - { - 'loss': loss.item(), - 'grad_norm': grad_norm.item(), - }, - global_step, - ) - return (new_optimizer_state, opt_update_fn), new_params, new_model_state - - -def prepare_for_eval( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params).""" - del workload - del hyperparameters - del current_params_types - del loss_type - del eval_results - del global_step - del rng - return (optimizer_state, current_param_container, model_state) - - -def get_batch_size(workload_name): - # Return the global batch size. - if workload_name == 'criteo1tb': - return 262_144 - elif workload_name == 'fastmri': - return 32 - elif workload_name == 'imagenet_resnet': - return 1024 - elif workload_name == 'imagenet_resnet_silu': - return 512 - elif workload_name == 'imagenet_resnet_gelu': - return 512 - elif workload_name == 'imagenet_vit': - return 1024 - elif workload_name == 'librispeech_conformer': - return 256 - elif workload_name == 'librispeech_deepspeech': - return 256 - elif workload_name == 'ogbg': - return 512 - elif workload_name == 'wmt': - return 128 - elif workload_name == 'mnist': - return 16 - else: - raise ValueError(f'Unsupported workload name: {workload_name}.') - - -def data_selection( - workload: spec.Workload, - input_queue: Iterator[Dict[str, spec.Tensor]], - optimizer_state: spec.OptimizerState, - current_param_container: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - global_step: int, - rng: spec.RandomState, -) -> Dict[str, spec.Tensor]: - """Select data from the infinitely repeating, pre-shuffled input queue. - Each element of the queue is a batch of training examples and labels. - """ - del workload - del optimizer_state - del current_param_container - del model_state - del hyperparameters - del global_step - del rng - batch = next(input_queue) - return batch diff --git a/reference_algorithms/qualification_baselines/external_tuning/pytorch_nadamw_target_setting.py b/reference_algorithms/qualification_baselines/external_tuning/pytorch_nadamw_target_setting.py deleted file mode 100644 index 8a2064bc0..000000000 --- a/reference_algorithms/qualification_baselines/external_tuning/pytorch_nadamw_target_setting.py +++ /dev/null @@ -1,400 +0,0 @@ -"""Submission file for an NAdamW optimizer with warmup+cosine LR in PyTorch.""" - -import math -from typing import Any, Dict, Iterator, List, Optional, Tuple - -import torch -import torch.distributed.nn as dist_nn -from absl import logging -from torch import Tensor -from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR - -from algoperf import spec -from algoperf.pytorch_utils import pytorch_setup - -USE_PYTORCH_DDP = pytorch_setup()[0] - - -# Modified from github.com/pytorch/pytorch/blob/v1.12.1/torch/optim/adamw.py. -class NAdamW(torch.optim.Optimizer): - r"""Implements NAdamW algorithm. - - See Table 1 in https://arxiv.org/abs/1910.05446 for the implementation of - the NAdam algorithm (there is also a comment in the code which highlights - the only difference of NAdamW and AdamW). - For further details regarding the algorithm we refer to - `Decoupled Weight Decay Regularization`_. - - Args: - params (iterable): iterable of parameters to optimize or dicts defining - parameter groups - lr (float, optional): learning rate (default: 1e-3) - betas (Tuple[float, float], optional): coefficients used for computing - running averages of gradient and its square (default: (0.9, 0.999)) - eps (float, optional): term added to the denominator to improve - numerical stability (default: 1e-8) - weight_decay (float, optional): weight decay coefficient (default: 1e-2) - .. _Decoupled Weight Decay Regularization: - https://arxiv.org/abs/1711.05101 - .. _On the Convergence of Adam and Beyond: - https://openreview.net/forum?id=ryQu7f-RZ - """ - - def __init__( - self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-2 - ): - if not 0.0 <= lr: - raise ValueError(f'Invalid learning rate: {lr}') - if not 0.0 <= eps: - raise ValueError(f'Invalid epsilon value: {eps}') - if not 0.0 <= betas[0] < 1.0: - raise ValueError(f'Invalid beta parameter at index 0: {betas[0]}') - if not 0.0 <= betas[1] < 1.0: - raise ValueError(f'Invalid beta parameter at index 1: {betas[1]}') - if not 0.0 <= weight_decay: - raise ValueError(f'Invalid weight_decay value: {weight_decay}') - defaults = { - 'lr': lr, - 'betas': betas, - 'eps': eps, - 'weight_decay': weight_decay, - } - super().__init__(params, defaults) - - def __setstate__(self, state): - super().__setstate__(state) - state_values = list(self.state.values()) - step_is_tensor = (len(state_values) != 0) and torch.is_tensor( - state_values[0]['step'] - ) - if not step_is_tensor: - for s in state_values: - s['step'] = torch.tensor(float(s['step'])) - - @torch.no_grad() - def step(self, closure=None): - """Performs a single optimization step. - - Args: - closure (callable, optional): A closure that reevaluates the model - and returns the loss. - """ - self._cuda_graph_capture_health_check() - - loss = None - if closure is not None: - with torch.enable_grad(): - loss = closure() - - for group in self.param_groups: - params_with_grad = [] - grads = [] - exp_avgs = [] - exp_avg_sqs = [] - state_steps = [] - beta1, beta2 = group['betas'] - - for p in group['params']: - if p.grad is None: - continue - params_with_grad.append(p) - if p.grad.is_sparse: - raise RuntimeError('NAdamW does not support sparse gradients') - grads.append(p.grad) - - state = self.state[p] - - # State initialization - if len(state) == 0: - state['step'] = torch.tensor(0.0) - # Exponential moving average of gradient values - state['exp_avg'] = torch.zeros_like( - p, memory_format=torch.preserve_format - ) - # Exponential moving average of squared gradient values - state['exp_avg_sq'] = torch.zeros_like( - p, memory_format=torch.preserve_format - ) - - exp_avgs.append(state['exp_avg']) - exp_avg_sqs.append(state['exp_avg_sq']) - state_steps.append(state['step']) - - nadamw( - params_with_grad, - grads, - exp_avgs, - exp_avg_sqs, - state_steps, - beta1=beta1, - beta2=beta2, - lr=group['lr'], - weight_decay=group['weight_decay'], - eps=group['eps'], - ) - - return loss - - -def nadamw( - params: List[Tensor], - grads: List[Tensor], - exp_avgs: List[Tensor], - exp_avg_sqs: List[Tensor], - state_steps: List[Tensor], - beta1: float, - beta2: float, - lr: float, - weight_decay: float, - eps: float, -) -> None: - r"""Functional API that performs NAdamW algorithm computation. - See NAdamW class for details. - """ - - if not all(isinstance(t, torch.Tensor) for t in state_steps): - raise RuntimeError( - 'API has changed, `state_steps` argument must contain a list of' - + ' singleton tensors' - ) - - for i, param in enumerate(params): - grad = grads[i] - exp_avg = exp_avgs[i] - exp_avg_sq = exp_avg_sqs[i] - step_t = state_steps[i] - - # Update step. - step_t += 1 - - # Perform stepweight decay. - param.mul_(1 - lr * weight_decay) - - # Decay the first and second moment running average coefficient. - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - - # Only difference between NAdamW and AdamW in this implementation. - # The official PyTorch implementation of NAdam uses a different algorithm. - # We undo these ops later on, which could cause numerical issues but saves - # us from having to make an extra copy of the gradients. - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - - step = step_t.item() - - bias_correction1 = 1 - beta1**step - bias_correction2 = 1 - beta2**step - - step_size = lr / bias_correction1 - - bias_correction2_sqrt = math.sqrt(bias_correction2) - denom = (exp_avg_sq.sqrt() / bias_correction2_sqrt).add_(eps) - - param.addcdiv_(exp_avg, denom, value=-step_size) - exp_avg.sub_(grad, alpha=1 - beta1).div_(beta1) - - -def init_optimizer_state( - workload: spec.Workload, - model_params: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - rng: spec.RandomState, -) -> spec.OptimizerState: - """Creates a NAdamW optimizer and a learning rate schedule.""" - del model_state - del rng - - optimizer_state = { - 'optimizer': NAdamW( - model_params.parameters(), - lr=hyperparameters.learning_rate, - betas=(1.0 - hyperparameters.one_minus_beta1, hyperparameters.beta2), - eps=1e-8, - weight_decay=hyperparameters.weight_decay, - ), - } - - def pytorch_cosine_warmup(step_hint: int, hyperparameters, optimizer): - warmup_steps = int(hyperparameters.warmup_factor * step_hint) - warmup = LinearLR( - optimizer, start_factor=1e-10, end_factor=1.0, total_iters=warmup_steps - ) - cosine_steps = max(step_hint - warmup_steps, 1) - cosine_decay = CosineAnnealingLR(optimizer, T_max=cosine_steps) - return SequentialLR( - optimizer, schedulers=[warmup, cosine_decay], milestones=[warmup_steps] - ) - - optimizer_state['scheduler'] = pytorch_cosine_warmup( - workload.step_hint * 0.75, hyperparameters, optimizer_state['optimizer'] - ) - - return optimizer_state - - -def update_params( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - batch: Dict[str, spec.Tensor], - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, - train_state: Optional[Dict[str, Any]] = None, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params, updated_model_state).""" - del current_params_types - del loss_type - del train_state - del eval_results - - current_model = current_param_container - current_model.train() - optimizer_state['optimizer'].zero_grad() - - logits_batch, new_model_state = workload.model_fn( - params=current_model, - augmented_and_preprocessed_input_batch=batch, - model_state=model_state, - mode=spec.ForwardPassMode.TRAIN, - rng=rng, - update_batch_norm=True, - dropout_rate=hyperparameters.dropout_rate, - ) - - label_smoothing = ( - hyperparameters.label_smoothing - if hasattr(hyperparameters, 'label_smoothing') - else 0.0 - ) - if hasattr(hyperparameters, 'grad_clip'): - grad_clip = hyperparameters.grad_clip - else: - grad_clip = None - - loss_dict = workload.loss_fn( - label_batch=batch['targets'], - logits_batch=logits_batch, - mask_batch=batch.get('weights'), - label_smoothing=label_smoothing, - ) - summed_loss = loss_dict['summed'] - n_valid_examples = loss_dict['n_valid_examples'] - if USE_PYTORCH_DDP: - # Use dist_nn.all_reduce to ensure correct loss and gradient scaling. - summed_loss = dist_nn.all_reduce(summed_loss) - n_valid_examples = dist_nn.all_reduce(n_valid_examples) - loss = summed_loss / n_valid_examples - - loss.backward() - - if grad_clip is not None: - torch.nn.utils.clip_grad_norm_( - current_model.parameters(), max_norm=grad_clip - ) - optimizer_state['optimizer'].step() - optimizer_state['scheduler'].step() - - # Log training metrics - loss, grad_norm, batch_size. - if global_step <= 100 or global_step % 500 == 0: - with torch.no_grad(): - parameters = [p for p in current_model.parameters() if p.grad is not None] - grad_norm = torch.norm( - torch.stack([torch.norm(p.grad.detach(), 2) for p in parameters]), 2 - ) - if workload.metrics_logger is not None: - workload.metrics_logger.append_scalar_metrics( - { - 'loss': loss.item(), - 'grad_norm': grad_norm.item(), - }, - global_step, - ) - logging.info( - '%d) loss = %0.3f, grad_norm = %0.3f', - global_step, - loss.item(), - grad_norm.item(), - ) - - return (optimizer_state, current_param_container, new_model_state) - - -def prepare_for_eval( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params).""" - del workload - del hyperparameters - del current_params_types - del loss_type - del eval_results - del global_step - del rng - return (optimizer_state, current_param_container, model_state) - - -def get_batch_size(workload_name): - # Return the global batch size. - if workload_name == 'criteo1tb': - return 262_144 - elif workload_name == 'fastmri': - return 32 - elif workload_name == 'imagenet_resnet': - return 1024 - elif workload_name == 'imagenet_resnet_silu': - return 512 - elif workload_name == 'imagenet_resnet_gelu': - return 512 - elif workload_name == 'imagenet_vit': - return 1024 - elif workload_name == 'librispeech_conformer': - return 256 - elif workload_name == 'librispeech_deepspeech': - return 256 - elif workload_name == 'ogbg': - return 512 - elif workload_name == 'wmt': - return 128 - elif workload_name == 'mnist': - return 16 - else: - raise ValueError(f'Unsupported workload name: {workload_name}.') - - -def data_selection( - workload: spec.Workload, - input_queue: Iterator[Dict[str, spec.Tensor]], - optimizer_state: spec.OptimizerState, - current_param_container: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - global_step: int, - rng: spec.RandomState, -) -> Dict[str, spec.Tensor]: - """Select data from the infinitely repeating, pre-shuffled input queue. - Each element of the queue is a batch of training examples and labels. - """ - del workload - del optimizer_state - del current_param_container - del model_state - del hyperparameters - del global_step - del rng - batch = next(input_queue) - return batch diff --git a/reference_algorithms/qualification_baselines/self_tuning/jax_nadamw_target_setting.py b/reference_algorithms/qualification_baselines/self_tuning/jax_nadamw_target_setting.py deleted file mode 100644 index 5a0ef5016..000000000 --- a/reference_algorithms/qualification_baselines/self_tuning/jax_nadamw_target_setting.py +++ /dev/null @@ -1,451 +0,0 @@ -"""Submission file for an NAdamW optimizer with warmup+cosine LR in Jax.""" - -# isort: off -# We have to turn off isort here to resolve a conflict between isort and yapf. -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - NamedTuple, - Optional, - Tuple, - Union, -) -# isort: on - -import chex -import jax -import jax.numpy as jnp -import optax -from jax import lax - -from algoperf import jax_sharding_utils, spec - -_GRAD_CLIP_EPS = 1e-6 - -HPARAMS = { - 'dropout_rate': 0.1, - 'learning_rate': 0.0017486387539278373, - 'one_minus_beta1': 0.06733926164, - 'beta2': 0.9955159689799007, - 'weight_decay': 0.08121616522670176, - 'warmup_factor': 0.02, -} - - -# Forked from -# github.com/google/init2winit/blob/master/init2winit/optimizer_lib/alias.py -def nadamw( - learning_rate: Union[float, optax.Schedule], - b1: float = 0.9, - b2: float = 0.999, - eps: float = 1e-8, - eps_root: float = 0.0, - debias: bool = True, - weight_decay: float = 0.0, - weight_decay_mask: Optional[Union[Any, Callable[[optax.Params], Any]]] = None, -) -> optax.GradientTransformation: - """Rescale updates according to the NAdam algorithm. - - References: - There seem to be multiple versions of NAdam. The original version is here - https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ (the official PyTorch - implementation also follows this). - Current code implements a simpler version with no momentum decay and slightly - different bias correction terms. The exact description can be found here - https://arxiv.org/pdf/1910.05446.pdf (Table 1). - - Args: - learning_rate: A fixed global scaling factor. - b1: Decay rate for the exponentially weighted average of grads. - b2: Decay rate for the exponentially weighted average of squared grads. - eps: Term added to the denominator to improve numerical stability. - eps_root: Term added to the denominator inside the square-root to improve - numerical stability when backpropagating gradients through the rescaling. - debias: Whether to use bias correction. - weight_decay: Strength of the weight decay regularization. Note that this - weight decay is multiplied with the learning rate. This is consistent with - other frameworks such as PyTorch, but different from (Loshchilov et al, - 2019) where the weight decay is only multiplied with the "schedule - multiplier", but not the base learning rate. - weight_decay_mask: A tree with same structure as (or a prefix of) the params - PyTree, or a Callable that returns such a pytree given the params/updates. - The leaves should be booleans, `True` for leaves/subtrees you want to - apply the weight decay to, and `False` for those you want to skip. Note - that the Nadam gradient transformations are applied to all parameters. - - Returns: - An (init_fn, update_fn) tuple. - """ - return optax.chain( - scale_by_nadam(b1, b2, eps, eps_root, debias), - optax.add_decayed_weights(weight_decay, weight_decay_mask), - scale_by_learning_rate(learning_rate), - ) - - -# All functions below are forked from -# github.com/google/init2winit/blob/master/init2winit/optimizer_lib/transform.py -def scale_by_nadam( - b1: float = 0.9, - b2: float = 0.999, - eps: float = 1e-8, - eps_root: float = 0.0, - debias: bool = True, - power: float = 0.5, -) -> optax.GradientTransformation: - """Rescale updates according to the NAdam algorithm. - - References: - There seem to be multiple versions of NAdam. The original version is here - https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ (the pytorch imp. also - follows this). - - Current code implements a simpler version with no momentum decay and slightly - different (standard Adam) bias correction terms. The exact description can be - found here https://arxiv.org/pdf/1910.05446.pdf (Table 1) - - Args: - b1: Decay rate for the exponentially weighted average of grads. - b2: Decay rate for the exponentially weighted average of squared grads. - eps: Term added to the denominator to improve numerical stability. - eps_root: Term added to the denominator inside the square-root to improve - numerical stability when backpropagating gradients through the rescaling. - debias: Whether to use bias correction. - power: The power to use in the preconditioner (0.5 in default adam). - Returns: - An (init_fn, update_fn) tuple. - """ - raise_power = jnp.sqrt if power == 0.5 else lambda x: jnp.power(x, power) - - def init_fn(params): - mu = jax.tree.map(jnp.zeros_like, params) # First moment - nu = jax.tree.map(jnp.zeros_like, params) # Second moment - return ScaleByAdamState(count=jnp.zeros([], jnp.int32), mu=mu, nu=nu) - - def update_fn(updates, state, params=None): - del params - mu = _update_moment(updates, state.mu, b1, 1) - nu = _update_moment(updates, state.nu, b2, 2) - count = state.count + jnp.array(1, dtype=jnp.int32) - mu_hat = _update_moment(updates, mu, b1, 1) - mu_hat = mu_hat if not debias else _bias_correction(mu_hat, b1, count) - nu_hat = nu if not debias else _bias_correction(nu, b2, count) - updates = jax.tree.map( - lambda m, v: m / (raise_power(v + eps_root) + eps), mu_hat, nu_hat - ) - return updates, ScaleByAdamState(count=count, mu=mu, nu=nu) - - return optax.GradientTransformation(init_fn, update_fn) - - -class ScaleByAdamState(NamedTuple): - """State for the NAdam algorithm.""" - - count: chex.Array # shape=(), dtype=jnp.int32. - mu: optax.Updates - nu: optax.Updates - - -def _update_moment(updates, moments, decay, order): - """Compute the exponential moving average of the `order-th` moment.""" - return jax.tree.map( - lambda g, t: (1 - decay) * (g**order) + decay * t, updates, moments - ) - - -def _bias_correction(moment, decay, count): - """Perform bias correction. This becomes a no-op as count goes to infinity.""" - beta = 1 - decay**count - return jax.tree.map(lambda t: t / beta.astype(t.dtype), moment) - - -def scale_by_learning_rate(learning_rate, flip_sign=True): - m = -1 if flip_sign else 1 - if callable(learning_rate): - return optax.scale_by_schedule(lambda count: m * learning_rate(count)) - return optax.scale(m * learning_rate) - - -def init_optimizer_state( - workload: spec.Workload, - model_params: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - rng: spec.RandomState, -) -> spec.OptimizerState: - """Creates a NAdamW optimizer and a learning rate schedule.""" - del model_params - del model_state - del rng - del hyperparameters - - hyperparameters = HPARAMS - - def jax_cosine_warmup(step_hint: int, hyperparameters): - # Create learning rate schedule. - warmup_steps = int(hyperparameters['warmup_factor'] * step_hint) - warmup_fn = optax.linear_schedule( - init_value=0.0, - end_value=hyperparameters['learning_rate'], - transition_steps=warmup_steps, - ) - cosine_steps = max(step_hint - warmup_steps, 1) - cosine_fn = optax.cosine_decay_schedule( - init_value=hyperparameters['learning_rate'], decay_steps=cosine_steps - ) - schedule_fn = optax.join_schedules( - schedules=[warmup_fn, cosine_fn], boundaries=[warmup_steps] - ) - return schedule_fn - - # Create optimizer + LR schedule. - lr_schedule_fn = jax_cosine_warmup(workload.step_hint * 0.75, hyperparameters) - opt_init_fn, opt_update_fn = nadamw( - learning_rate=lr_schedule_fn, - b1=1.0 - hyperparameters['one_minus_beta1'], - b2=hyperparameters['beta2'], - eps=1e-8, - weight_decay=hyperparameters['weight_decay'], - ) - params_zeros_like = jax.tree.map( - lambda s: jnp.zeros(s.shape_tuple), workload.param_shapes - ) - optimizer_state = opt_init_fn(params_zeros_like) - - return optimizer_state, opt_update_fn - - -def train_step( - workload, - opt_update_fn, - model_state, - optimizer_state, - current_param_container, - batch, - rng, - grad_clip, - label_smoothing, - dropout_rate, -): - def _loss_fn(params): - """Loss function used for training.""" - logits, new_model_state = workload.model_fn( - params, - batch, - model_state, - spec.ForwardPassMode.TRAIN, - rng, - update_batch_norm=True, - dropout_rate=dropout_rate, - ) - loss_dict = workload.loss_fn( - label_batch=batch['targets'], - logits_batch=logits, - mask_batch=batch.get('weights'), - label_smoothing=label_smoothing, - ) - summed_loss = loss_dict['summed'] - n_valid_examples = loss_dict['n_valid_examples'] - return summed_loss, (n_valid_examples, new_model_state) - - grad_fn = jax.value_and_grad(_loss_fn, has_aux=True) - (summed_loss, (n_valid_examples, new_model_state)), grad = grad_fn( - current_param_container - ) - # Get correct global mean loss and grad. - (summed_loss, n_valid_examples, grad) = lax.psum( - (summed_loss, n_valid_examples, grad), axis_name='batch' - ) - loss = summed_loss / n_valid_examples - grad = jax.tree.map(lambda x: x / n_valid_examples, grad) - - grad_norm = jnp.sqrt( - sum(jnp.sum(g**2) for g in jax.tree_util.tree_leaves(grad)) - ) - - if grad_clip is not None: - grad_scaling_factor = grad_clip / (grad_norm + _GRAD_CLIP_EPS) - grad_scaling_factor = jax.lax.clamp(min=0.0, x=grad_scaling_factor, max=1.0) - grad = jax.tree.map(lambda x: x * grad_scaling_factor, grad) - - updates, new_optimizer_state = opt_update_fn( - grad, optimizer_state, current_param_container - ) - updated_params = optax.apply_updates(current_param_container, updates) - return new_optimizer_state, updated_params, new_model_state, loss, grad_norm - - -def update_params( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - batch: Dict[str, spec.Tensor], - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, - train_state: Optional[Dict[str, Any]] = None, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params, updated_model_state).""" - del current_params_types - del loss_type - del train_state - del eval_results - del hyperparameters - - hyperparameters = HPARAMS - - optimizer_state, opt_update_fn = optimizer_state - if hasattr(hyperparameters, 'label_smoothing'): - label_smoothing = hyperparameters['label_smoothing'] - else: - label_smoothing = 0.0 - if hasattr(hyperparameters, 'grad_clip'): - grad_clip = hyperparameters['grad_clip'] - else: - grad_clip = None - dropout_rate = hyperparameters.dropout_rate - - # Create shardings for each argument - mesh = jax.sharding.Mesh(jax.devices(), ('batch')) - replicated = jax_sharding_utils.get_replicate_sharding( - mesh - ) # No partitioning - sharded = jax_sharding_utils.get_batch_sharding( - mesh - ) # Partition along batch dimension - - # Create the sharding rules for each argument - arg_shardings = ( - # workload is static - # opt_update_fn is static - replicated, # model_state - replicated, # optimizer_state - replicated, # current_param_container - sharded, # batch - replicated, # rng - replicated, # grad_clip - replicated, # label_smoothing - replicated, # dropout_rate - ) - out_shardings = ( - replicated, # new_optimizer_state - replicated, # updated_params - replicated, # new_model_state - replicated, # loss - replicated, # grad_norm - ) - - # Jit with shardings - jitted_train_step = jax.jit( - train_step, - static_argnums=(0, 1), - donate_argnums=(2, 3, 4), - in_shardings=arg_shardings, - out_shardings=out_shardings, - ) - outputs = jitted_train_step( - workload, - opt_update_fn, - model_state, - optimizer_state, - current_param_container, - batch, - rng, - grad_clip, - label_smoothing, - dropout_rate, - ) - new_optimizer_state, new_params, new_model_state, loss, grad_norm = outputs - - # Log loss, grad_norm. - if global_step % 100 == 0 and workload.metrics_logger is not None: - workload.metrics_logger.append_scalar_metrics( - { - 'loss': loss[0], - 'grad_norm': grad_norm[0], - }, - global_step, - ) - return (new_optimizer_state, opt_update_fn), new_params, new_model_state - - -def prepare_for_eval( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params).""" - del workload - del hyperparameters - del current_params_types - del loss_type - del eval_results - del global_step - del rng - return (optimizer_state, current_param_container, model_state) - - -def get_batch_size(workload_name): - # Return the global batch size. - if workload_name == 'criteo1tb': - return 262_144 - elif workload_name == 'fastmri': - return 32 - elif workload_name == 'imagenet_resnet': - return 1024 - elif workload_name == 'imagenet_resnet_silu': - return 512 - elif workload_name == 'imagenet_resnet_gelu': - return 512 - elif workload_name == 'imagenet_vit': - return 1024 - elif workload_name == 'librispeech_conformer': - return 256 - elif workload_name == 'librispeech_deepspeech': - return 256 - elif workload_name == 'ogbg': - return 512 - elif workload_name == 'wmt': - return 128 - elif workload_name == 'mnist': - return 16 - else: - raise ValueError(f'Unsupported workload name: {workload_name}.') - - -def data_selection( - workload: spec.Workload, - input_queue: Iterator[Dict[str, spec.Tensor]], - optimizer_state: spec.OptimizerState, - current_param_container: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - global_step: int, - rng: spec.RandomState, -) -> Dict[str, spec.Tensor]: - """Select data from the infinitely repeating, pre-shuffled input queue. - Each element of the queue is a batch of training examples and labels. - """ - del workload - del optimizer_state - del current_param_container - del model_state - del hyperparameters - del global_step - del rng - batch = next(input_queue) - return batch diff --git a/reference_algorithms/qualification_baselines/self_tuning/pytorch_nadamw_target_setting.py b/reference_algorithms/qualification_baselines/self_tuning/pytorch_nadamw_target_setting.py deleted file mode 100644 index acf7c72ec..000000000 --- a/reference_algorithms/qualification_baselines/self_tuning/pytorch_nadamw_target_setting.py +++ /dev/null @@ -1,417 +0,0 @@ -"""Submission file for an NAdamW optimizer with warmup+cosine LR in PyTorch.""" - -import collections -import math -from typing import Any, Dict, Iterator, List, Optional, Tuple - -import torch -import torch.distributed.nn as dist_nn -from absl import logging -from torch import Tensor -from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR - -from algoperf import spec -from algoperf.pytorch_utils import pytorch_setup - -USE_PYTORCH_DDP = pytorch_setup()[0] - -HPARAMS = { - 'dropout_rate': 0.1, - 'learning_rate': 0.0017486387539278373, - 'one_minus_beta1': 0.06733926164, - 'beta2': 0.9955159689799007, - 'weight_decay': 0.08121616522670176, - 'warmup_factor': 0.02, -} -HPARAMS = collections.namedtuple('Hyperparameters', HPARAMS.keys())(**HPARAMS) - - -# Modified from github.com/pytorch/pytorch/blob/v1.12.1/torch/optim/adamw.py. -class NAdamW(torch.optim.Optimizer): - r"""Implements NAdamW algorithm. - - See Table 1 in https://arxiv.org/abs/1910.05446 for the implementation of - the NAdam algorithm (there is also a comment in the code which highlights - the only difference of NAdamW and AdamW). - For further details regarding the algorithm we refer to - `Decoupled Weight Decay Regularization`_. - - Args: - params (iterable): iterable of parameters to optimize or dicts defining - parameter groups - lr (float, optional): learning rate (default: 1e-3) - betas (Tuple[float, float], optional): coefficients used for computing - running averages of gradient and its square (default: (0.9, 0.999)) - eps (float, optional): term added to the denominator to improve - numerical stability (default: 1e-8) - weight_decay (float, optional): weight decay coefficient (default: 1e-2) - .. _Decoupled Weight Decay Regularization: - https://arxiv.org/abs/1711.05101 - .. _On the Convergence of Adam and Beyond: - https://openreview.net/forum?id=ryQu7f-RZ - """ - - def __init__( - self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-2 - ): - if not 0.0 <= lr: - raise ValueError(f'Invalid learning rate: {lr}') - if not 0.0 <= eps: - raise ValueError(f'Invalid epsilon value: {eps}') - if not 0.0 <= betas[0] < 1.0: - raise ValueError(f'Invalid beta parameter at index 0: {betas[0]}') - if not 0.0 <= betas[1] < 1.0: - raise ValueError(f'Invalid beta parameter at index 1: {betas[1]}') - if not 0.0 <= weight_decay: - raise ValueError(f'Invalid weight_decay value: {weight_decay}') - defaults = { - 'lr': lr, - 'betas': betas, - 'eps': eps, - 'weight_decay': weight_decay, - } - super().__init__(params, defaults) - - def __setstate__(self, state): - super().__setstate__(state) - state_values = list(self.state.values()) - step_is_tensor = (len(state_values) != 0) and torch.is_tensor( - state_values[0]['step'] - ) - if not step_is_tensor: - for s in state_values: - s['step'] = torch.tensor(float(s['step'])) - - @torch.no_grad() - def step(self, closure=None): - """Performs a single optimization step. - - Args: - closure (callable, optional): A closure that reevaluates the model - and returns the loss. - """ - self._cuda_graph_capture_health_check() - - loss = None - if closure is not None: - with torch.enable_grad(): - loss = closure() - - for group in self.param_groups: - params_with_grad = [] - grads = [] - exp_avgs = [] - exp_avg_sqs = [] - state_steps = [] - beta1, beta2 = group['betas'] - - for p in group['params']: - if p.grad is None: - continue - params_with_grad.append(p) - if p.grad.is_sparse: - raise RuntimeError('NAdamW does not support sparse gradients') - grads.append(p.grad) - - state = self.state[p] - - # State initialization - if len(state) == 0: - state['step'] = torch.tensor(0.0) - # Exponential moving average of gradient values - state['exp_avg'] = torch.zeros_like( - p, memory_format=torch.preserve_format - ) - # Exponential moving average of squared gradient values - state['exp_avg_sq'] = torch.zeros_like( - p, memory_format=torch.preserve_format - ) - - exp_avgs.append(state['exp_avg']) - exp_avg_sqs.append(state['exp_avg_sq']) - state_steps.append(state['step']) - - nadamw( - params_with_grad, - grads, - exp_avgs, - exp_avg_sqs, - state_steps, - beta1=beta1, - beta2=beta2, - lr=group['lr'], - weight_decay=group['weight_decay'], - eps=group['eps'], - ) - - return loss - - -def nadamw( - params: List[Tensor], - grads: List[Tensor], - exp_avgs: List[Tensor], - exp_avg_sqs: List[Tensor], - state_steps: List[Tensor], - beta1: float, - beta2: float, - lr: float, - weight_decay: float, - eps: float, -) -> None: - r"""Functional API that performs NAdamW algorithm computation. - See NAdamW class for details. - """ - - if not all(isinstance(t, torch.Tensor) for t in state_steps): - raise RuntimeError( - 'API has changed, `state_steps` argument must contain a list of' - + ' singleton tensors' - ) - - for i, param in enumerate(params): - grad = grads[i] - exp_avg = exp_avgs[i] - exp_avg_sq = exp_avg_sqs[i] - step_t = state_steps[i] - - # Update step. - step_t += 1 - - # Perform stepweight decay. - param.mul_(1 - lr * weight_decay) - - # Decay the first and second moment running average coefficient. - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - - # Only difference between NAdamW and AdamW in this implementation. - # The official PyTorch implementation of NAdam uses a different algorithm. - # We undo these ops later on, which could cause numerical issues but saves - # us from having to make an extra copy of the gradients. - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - - step = step_t.item() - - bias_correction1 = 1 - beta1**step - bias_correction2 = 1 - beta2**step - - step_size = lr / bias_correction1 - - bias_correction2_sqrt = math.sqrt(bias_correction2) - denom = (exp_avg_sq.sqrt() / bias_correction2_sqrt).add_(eps) - - param.addcdiv_(exp_avg, denom, value=-step_size) - exp_avg.sub_(grad, alpha=1 - beta1).div_(beta1) - - -def init_optimizer_state( - workload: spec.Workload, - model_params: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - rng: spec.RandomState, -) -> spec.OptimizerState: - """Creates a NAdamW optimizer and a learning rate schedule.""" - del model_state - del rng - del hyperparameters - - hyperparameters = HPARAMS - - optimizer_state = { - 'optimizer': NAdamW( - model_params.parameters(), - lr=hyperparameters.learning_rate, - betas=(1.0 - hyperparameters.one_minus_beta1, hyperparameters.beta2), - eps=1e-8, - weight_decay=hyperparameters.weight_decay, - ), - } - - def pytorch_cosine_warmup(step_hint: int, hyperparameters, optimizer): - warmup_steps = int(hyperparameters.warmup_factor * step_hint) - warmup = LinearLR( - optimizer, start_factor=1e-10, end_factor=1.0, total_iters=warmup_steps - ) - cosine_steps = max(step_hint - warmup_steps, 1) - cosine_decay = CosineAnnealingLR(optimizer, T_max=cosine_steps) - return SequentialLR( - optimizer, schedulers=[warmup, cosine_decay], milestones=[warmup_steps] - ) - - optimizer_state['scheduler'] = pytorch_cosine_warmup( - workload.step_hint * 0.75, hyperparameters, optimizer_state['optimizer'] - ) - - return optimizer_state - - -def update_params( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - batch: Dict[str, spec.Tensor], - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, - train_state: Optional[Dict[str, Any]] = None, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params, updated_model_state).""" - del current_params_types - del loss_type - del train_state - del eval_results - del hyperparameters - - hyperparameters = HPARAMS - - current_model = current_param_container - current_model.train() - optimizer_state['optimizer'].zero_grad() - - logits_batch, new_model_state = workload.model_fn( - params=current_model, - augmented_and_preprocessed_input_batch=batch, - model_state=model_state, - mode=spec.ForwardPassMode.TRAIN, - rng=rng, - update_batch_norm=True, - dropout_rate=hyperparameters.dropout_rate, - ) - - label_smoothing = ( - hyperparameters.label_smoothing - if hasattr(hyperparameters, 'label_smoothing') - else 0.0 - ) - if hasattr(hyperparameters, 'grad_clip'): - grad_clip = hyperparameters.grad_clip - else: - grad_clip = None - - loss_dict = workload.loss_fn( - label_batch=batch['targets'], - logits_batch=logits_batch, - mask_batch=batch.get('weights'), - label_smoothing=label_smoothing, - ) - summed_loss = loss_dict['summed'] - n_valid_examples = loss_dict['n_valid_examples'] - if USE_PYTORCH_DDP: - # Use dist_nn.all_reduce to ensure correct loss and gradient scaling. - summed_loss = dist_nn.all_reduce(summed_loss) - n_valid_examples = dist_nn.all_reduce(n_valid_examples) - loss = summed_loss / n_valid_examples - - loss.backward() - - if grad_clip is not None: - torch.nn.utils.clip_grad_norm_( - current_model.parameters(), max_norm=grad_clip - ) - optimizer_state['optimizer'].step() - optimizer_state['scheduler'].step() - - # Log training metrics - loss, grad_norm, batch_size. - if global_step <= 100 or global_step % 500 == 0: - with torch.no_grad(): - parameters = [p for p in current_model.parameters() if p.grad is not None] - grad_norm = torch.norm( - torch.stack([torch.norm(p.grad.detach(), 2) for p in parameters]), 2 - ) - if workload.metrics_logger is not None: - workload.metrics_logger.append_scalar_metrics( - { - 'loss': loss.item(), - 'grad_norm': grad_norm.item(), - }, - global_step, - ) - logging.info( - '%d) loss = %0.3f, grad_norm = %0.3f', - global_step, - loss.item(), - grad_norm.item(), - ) - - return (optimizer_state, current_param_container, new_model_state) - - -def prepare_for_eval( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, -) -> spec.UpdateReturn: - """Return (updated_optimizer_state, updated_params).""" - del workload - del hyperparameters - del current_params_types - del loss_type - del eval_results - del global_step - del rng - return (optimizer_state, current_param_container, model_state) - - -def get_batch_size(workload_name): - # Return the global batch size. - if workload_name == 'criteo1tb': - return 262_144 - elif workload_name == 'fastmri': - return 32 - elif workload_name == 'imagenet_resnet': - return 1024 - elif workload_name == 'imagenet_resnet_silu': - return 512 - elif workload_name == 'imagenet_resnet_gelu': - return 512 - elif workload_name == 'imagenet_vit': - return 1024 - elif workload_name == 'librispeech_conformer': - return 256 - elif workload_name == 'librispeech_deepspeech': - return 256 - elif workload_name == 'ogbg': - return 512 - elif workload_name == 'wmt': - return 128 - elif workload_name == 'mnist': - return 16 - else: - raise ValueError(f'Unsupported workload name: {workload_name}.') - - -def data_selection( - workload: spec.Workload, - input_queue: Iterator[Dict[str, spec.Tensor]], - optimizer_state: spec.OptimizerState, - current_param_container: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - global_step: int, - rng: spec.RandomState, -) -> Dict[str, spec.Tensor]: - """Select data from the infinitely repeating, pre-shuffled input queue. - Each element of the queue is a batch of training examples and labels. - """ - del workload - del optimizer_state - del current_param_container - del model_state - del hyperparameters - del global_step - del rng - batch = next(input_queue) - return batch diff --git a/scoring/algoperf_v05/score_submissions.py b/scoring/algoperf_v05/score_submissions.py index 6ef931a54..d7bda25ad 100644 --- a/scoring/algoperf_v05/score_submissions.py +++ b/scoring/algoperf_v05/score_submissions.py @@ -7,7 +7,7 @@ Example usage: python3 score_submissions.py \ - --submission_directory $HOME/algorithmic-efficiency/prize_qualification_baselines/logs \ + --submission_directory $HOME/algorithmic-efficiency/algorithms/baseline/logs \ --strict True --compute_performance_profiles """ diff --git a/scoring/utils/run_workloads.py b/scoring/utils/run_workloads.py index e2de01130..273881c5a 100644 --- a/scoring/utils/run_workloads.py +++ b/scoring/utils/run_workloads.py @@ -44,12 +44,12 @@ flags.DEFINE_boolean('local', False, 'Mount local algorithmic-efficiency repo.') flags.DEFINE_string( 'submission_path', - 'prize_qualification_baselines/external_tuning/jax_nadamw_full_budget.py', + 'algorithms/baseline/external_tuning/jax_nadamw_full_budget.py', 'Path to reference submission.', ) flags.DEFINE_string( 'tuning_search_space', - 'prize_qualification_baselines/external_tuning/tuning_search_space.json', + 'algorithms/baseline/external_tuning/tuning_search_space.json', 'Path to tuning search space.', ) flags.DEFINE_string('framework', 'jax', 'Can be either PyTorch or JAX.') @@ -65,7 +65,7 @@ enum_values=['external', 'self'], help='Can be either external of self.', ) -flags.DEFINE_integer('num_studies', 5, 'Number of studies to run') +flags.DEFINE_integer('num_studies', 3, 'Number of studies to run') flags.DEFINE_integer('study_start_index', None, 'Start index for studies.') flags.DEFINE_integer('study_end_index', None, 'End index for studies.') flags.DEFINE_integer('num_tuning_trials', 5, 'Number of tuning trials.') diff --git a/submission_runner.py b/submission_runner.py index d6149f3e9..552c99b79 100644 --- a/submission_runner.py +++ b/submission_runner.py @@ -6,9 +6,9 @@ python3 submission_runner.py \ --workload=mnist \ --framework=jax \ - --submission_path=reference_algorithms/development_algorithms/mnist/mnist_jax/submission.py \ + --submission_path=algorithms/development_algorithms/mnist/mnist_jax/submission.py \ --tuning_ruleset=external \ - --tuning_search_space=reference_algorithms/development_algorithms/mnist/tuning_search_space.json \ + --tuning_search_space=algorithms/development_algorithms/mnist/tuning_search_space.json \ --num_tuning_trials=3 \ --experiment_dir=/home/znado/experiment_dir \ --experiment_name=baseline diff --git a/submissions/template/submission.py b/submissions/template/submission.py deleted file mode 100644 index db6900afd..000000000 --- a/submissions/template/submission.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Template submission module. - -See https://github.com/mlcommons/algorithmic-efficiency/blob/main/DOCUMENTATION.md#allowed-submissions -and https://github.com/mlcommons/algorithmic-efficiency/blob/main/DOCUMENTATION.md#disallowed-submissions -for guidelines. -""" - -from typing import Any, Dict, Iterator, List, Optional, Tuple - -from algoperf import spec - - -def init_optimizer_state( - workload: spec.Workload, - model_params: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - rng: spec.RandomState, -) -> spec.OptimizerState: - """Creates a Nesterov optimizer and a learning rate schedule. - Returns: spec.OptimizerState initialized optimizer state - """ - pass - - -def update_params( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - batch: Dict[str, spec.Tensor], - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, - train_state: Optional[Dict[str, Any]] = None, -) -> spec.UpdateReturn: - """ - Returns: - spec.OptimizerState: new optimizer state - spec.ParameterTypeTree: new params - new_model_state: new model state - """ - pass - - -def prepare_for_eval( - workload: spec.Workload, - current_param_container: spec.ParameterContainer, - current_params_types: spec.ParameterTypeTree, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - loss_type: spec.LossType, - optimizer_state: spec.OptimizerState, - eval_results: List[Tuple[int, float]], - global_step: int, - rng: spec.RandomState, -) -> spec.UpdateReturn: - """ - Returns: - new_optimizer_state - new_params - new_model_state - """ - pass - - -def get_batch_size(workload_name): - """ - Gets batch size for workload. - Note that these batch sizes only apply during training and not during evals. - Args: - workload_name (str): Valid workload_name values are: "wmt", "ogbg", - "criteo1tb", "fastmri", "imagenet_resnet", "imagenet_vit", - "librispeech_deepspeech", "librispeech_conformer" or any of the - variants. - Returns: - int: batch_size - Raises: - ValueError: If workload_name is not handled. - """ - pass - - -def data_selection( - workload: spec.Workload, - input_queue: Iterator[Dict[str, spec.Tensor]], - optimizer_state: spec.OptimizerState, - current_param_container: spec.ParameterContainer, - model_state: spec.ModelAuxiliaryState, - hyperparameters: spec.Hyperparameters, - global_step: int, - rng: spec.RandomState, -) -> Dict[str, spec.Tensor]: - """Select data from the infinitely repeating, pre-shuffled input queue. - Each element of the queue is a batch of training examples and labels. - Tip: - If you would just like the next batch from the input queue return next(input_queue). - - Returns: - batch: next batch of input data - """ - pass diff --git a/tests/modeldiffs/vanilla_sgd_jax.py b/tests/modeldiffs/vanilla_sgd_jax.py index aa7bebd4f..ace3910d0 100644 --- a/tests/modeldiffs/vanilla_sgd_jax.py +++ b/tests/modeldiffs/vanilla_sgd_jax.py @@ -4,10 +4,10 @@ from flax import jax_utils from algoperf import spec -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.jax_submission_base import ( # noqa: F401 update_params, ) diff --git a/tests/modeldiffs/vanilla_sgd_pytorch.py b/tests/modeldiffs/vanilla_sgd_pytorch.py index 6448ac097..b1abaf192 100644 --- a/tests/modeldiffs/vanilla_sgd_pytorch.py +++ b/tests/modeldiffs/vanilla_sgd_pytorch.py @@ -1,10 +1,10 @@ import torch from algoperf import spec -from reference_algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 +from algorithms.target_setting_algorithms.data_selection import ( # noqa: F401 data_selection, ) -from reference_algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 +from algorithms.target_setting_algorithms.pytorch_submission_base import ( # noqa: F401 update_params, ) diff --git a/tests/reference_algorithm_tests.py b/tests/reference_algorithm_tests.py index 431104dae..defde9447 100644 --- a/tests/reference_algorithm_tests.py +++ b/tests/reference_algorithm_tests.py @@ -9,15 +9,15 @@ Assumes that each reference submission is using the external tuning ruleset and that it is defined in: # pylint: disable=line-too-long -"reference_algorithms/target_setting_algorithms/{workload}/{workload}_{framework}/submission.py" -"reference_algorithms/target_setting_algorithms/{workload}/tuning_search_space.json". +"algorithms/target_setting_algorithms/{workload}/{workload}_{framework}/submission.py" +"algorithms/target_setting_algorithms/{workload}/tuning_search_space.json". python3 tests/reference_algorithm_tests.py \ --workload=criteo1tb \ --framework=jax \ --global_batch_size=16 \ - --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py \ - --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json + --submission_path=algorithms/target_setting_algorithms/jax_adamw.py \ + --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json """ import copy @@ -526,12 +526,11 @@ def _make_paths(repo_location, framework, workload_name): else: dataset_name = workload_name workload_dir = ( - f'{repo_location}/reference_algorithms/target_setting_algorithms/' - f'{workload_name}' + f'{repo_location}/algorithms/target_setting_algorithms/{workload_name}' ) search_space_path = f'{workload_dir}/tuning_search_space.json' submission_path = ( - f'reference_algorithms/target_setting_algorithms/' + f'algorithms/target_setting_algorithms/' f'{workload_name}/{dataset_name}_{framework}/' 'submission.py' ) @@ -563,9 +562,7 @@ def test_submission(self): raise ValueError('Cannot set --submission_path and --all.') if FLAGS.tuning_search_space: raise ValueError('Cannot set --tuning_search_space and --all.') - references_dir = ( - f'{repo_location}/reference_algorithms/target_setting_algorithms' - ) + references_dir = f'{repo_location}/algorithms/target_setting_algorithms' for workload_name in os.listdir(references_dir): for framework in ['jax', 'pytorch']: if framework == 'pytorch': diff --git a/tests/test_baselines.py b/tests/test_baselines.py index e7ca736b8..051320ac7 100644 --- a/tests/test_baselines.py +++ b/tests/test_baselines.py @@ -42,7 +42,7 @@ 'jax', ] -baseline_path = 'reference_algorithms/paper_baselines' +baseline_path = 'algorithms/archived_paper_baselines' named_parameters = [] for f in frameworks: