From d73266355395509c3597172779bcab8f108014f7 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 13 Jul 2020 19:19:13 +0000 Subject: [PATCH 1/5] Add macos unittest --- .circleci/config.yml | 55 +++++++++++++++++++++++++++++++++++++++++ .circleci/config.yml.in | 40 ++++++++++++++++++++++++++++++ .circleci/regenerate.py | 5 +++- 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d46ff914f..d505053b28 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -533,6 +533,46 @@ jobs: - store_test_results: path: test-results + unittest_macos_cpu: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - install_build_tools_macos + - attach_workspace: + at: third_party + - generate_cache_key + - restore_cache: + + keys: + - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} + + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} + + paths: + - conda + - env + - third_party/build + - third_party/install + - third_party/src + - run: + name: Install torchaudio + command: .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results + stylecheck: <<: *binary_common docker: @@ -744,6 +784,21 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_windows_gpu_py3.8 python_version: '3.8' + - unittest_macos_cpu: + name: unittest_macos_cpu_py3.6 + python_version: '3.6' + requires: + - download_third_parties_nix + - unittest_macos_cpu: + name: unittest_macos_cpu_py3.7 + python_version: '3.7' + requires: + - download_third_parties_nix + - unittest_macos_cpu: + name: unittest_macos_cpu_py3.8 + python_version: '3.8' + requires: + - download_third_parties_nix nightly: jobs: - circleci_consistency: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 800eb56e27..b6048fc56e 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -533,6 +533,46 @@ jobs: - store_test_results: path: test-results + unittest_macos_cpu: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - install_build_tools_macos + - attach_workspace: + at: third_party + - generate_cache_key + - restore_cache: + {% raw %} + keys: + - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} + {% endraw %} + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + {% raw %} + key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} + {% endraw %} + paths: + - conda + - env + - third_party/build + - third_party/install + - third_party/src + - run: + name: Install torchaudio + command: .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results + stylecheck: <<: *binary_common docker: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 3f8b9f8676..2c9fe7ee4f 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -136,8 +136,11 @@ def indent(indentation, data_list): def unittest_workflows(indentation=6): jobs = [] jobs += build_download_job(None) - for os_type in ["linux", "windows"]: + for os_type in ["linux", "windows", "macos"]: for device_type in ["cpu", "gpu"]: + if os_type == "macos" and device_type == "gpu": + continue + for i, python_version in enumerate(PYTHON_VERSIONS): job = { "name": f"unittest_{os_type}_{device_type}_py{python_version}", From 59e8f11b4ad713aa5647febb469365ea0d185918 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 13 Jul 2020 19:32:29 +0000 Subject: [PATCH 2/5] Add wget --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d505053b28..6972d2c026 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ commands: steps: - run: name: Install cmake and pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget # Disable brew auto update which is very slow binary_common: &binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index b6048fc56e..cf71baf7dc 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -43,7 +43,7 @@ commands: steps: - run: name: Install cmake and pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget # Disable brew auto update which is very slow binary_common: &binary_common From 13625ef895d48c03c81a2725820e1c315492b444 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 13 Jul 2020 20:07:08 +0000 Subject: [PATCH 3/5] Update conda installer --- .circleci/unittest/linux/scripts/setup_env.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 489094a3b9..f9224ea2e6 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -17,7 +17,11 @@ cd "${root_dir}" # 1. Install conda at ./conda if [ ! -d "${conda_dir}" ]; then printf "* Installing conda\n" - wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux + esac + wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" bash ./miniconda.sh -b -f -p "${conda_dir}" fi eval "$(${conda_dir}/bin/conda shell.bash hook)" From e40a6f8189d195c2960dd5f66bf348f0368741a7 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 13 Jul 2020 20:25:55 +0000 Subject: [PATCH 4/5] Install clang-format separately --- .circleci/unittest/linux/scripts/environment.yml | 1 - .circleci/unittest/linux/scripts/setup_env.sh | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index 1a0cb1bf11..108d97d193 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -11,7 +11,6 @@ dependencies: - llvmlite==0.31 # See https://github.com/pytorch/audio/pull/766 - pip - pip: - - clang-format - kaldi-io - scipy - parameterized diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index f9224ea2e6..290f0f8f97 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -14,13 +14,14 @@ env_dir="${root_dir}/env" cd "${root_dir}" +case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux +esac + # 1. Install conda at ./conda if [ ! -d "${conda_dir}" ]; then printf "* Installing conda\n" - case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux - esac wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" bash ./miniconda.sh -b -f -p "${conda_dir}" fi @@ -36,6 +37,9 @@ conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune +if [ "${os}" == Linux ] ; then + pip install clang-format +fi # 4. Buld codecs mkdir -p third_party/build From 2f3c1f1f8f4508d9dcd97b7e620ef9b67930d5e8 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 13 Jul 2020 21:04:19 +0000 Subject: [PATCH 5/5] Set resource class to large for faster test execution --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6972d2c026..771dccc515 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -537,6 +537,7 @@ jobs: <<: *binary_common macos: xcode: "9.0" + resource_class: large steps: - checkout - install_build_tools_macos diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index cf71baf7dc..6d6d77d50f 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -537,6 +537,7 @@ jobs: <<: *binary_common macos: xcode: "9.0" + resource_class: large steps: - checkout - install_build_tools_macos