GH-48594: [C++][FlightRPC] Fix ODBC CI Long Build Time Issue #3904
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: C++ Extra | |
| on: | |
| push: | |
| branches: | |
| - '**' | |
| - '!dependabot/**' | |
| paths: | |
| - '.dockerignore' | |
| - '.github/workflows/check_labels.yml' | |
| - '.github/workflows/cpp_extra.yml' | |
| - '.github/workflows/report_ci.yml' | |
| - 'ci/conda_env_*' | |
| - 'ci/docker/**' | |
| - 'ci/scripts/ccache_setup.sh' | |
| - 'ci/scripts/cpp_*' | |
| - 'ci/scripts/install_azurite.sh' | |
| - 'ci/scripts/install_gcs_testbench.sh' | |
| - 'ci/scripts/install_minio.sh' | |
| - 'ci/scripts/msys2_*' | |
| - 'ci/scripts/util_*' | |
| - 'cpp/**' | |
| - 'compose.yaml' | |
| - 'format/Flight.proto' | |
| - 'testing' | |
| tags: | |
| - '**' | |
| pull_request: | |
| paths: | |
| - '.dockerignore' | |
| - '.github/workflows/check_labels.yml' | |
| - '.github/workflows/cpp_extra.yml' | |
| - '.github/workflows/report_ci.yml' | |
| - 'ci/conda_env_*' | |
| - 'ci/docker/**' | |
| - 'ci/scripts/ccache_setup.sh' | |
| - 'ci/scripts/cpp_*' | |
| - 'ci/scripts/install_azurite.sh' | |
| - 'ci/scripts/install_gcs_testbench.sh' | |
| - 'ci/scripts/install_minio.sh' | |
| - 'ci/scripts/msys2_*' | |
| - 'ci/scripts/util_*' | |
| - 'cpp/**' | |
| - 'compose.yaml' | |
| - 'format/Flight.proto' | |
| - 'testing' | |
| types: | |
| - labeled | |
| - opened | |
| - reopened | |
| - synchronize | |
| schedule: | |
| - cron: | | |
| 0 0 * * * | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| check-labels: | |
| if: github.event_name != 'schedule' || github.repository == 'apache/arrow' | |
| uses: ./.github/workflows/check_labels.yml | |
| secrets: inherit | |
| with: | |
| parent-workflow: cpp_extra | |
| docker: | |
| needs: check-labels | |
| name: ${{ matrix.title }} | |
| runs-on: ${{ matrix.runs-on }} | |
| if: >- | |
| needs.check-labels.outputs.force == 'true' || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') | |
| timeout-minutes: 75 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - image: alpine-linux-cpp | |
| runs-on: ubuntu-latest | |
| title: AMD64 Alpine Linux | |
| - image: conda-cpp | |
| run-options: >- | |
| -e ARROW_USE_MESON=ON | |
| runs-on: ubuntu-latest | |
| title: AMD64 Ubuntu Meson | |
| # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved | |
| - continue-on-error: true | |
| envs: | |
| - DEBIAN=13 | |
| image: debian-cpp | |
| run-options: >- | |
| -e CMAKE_CXX_STANDARD=23 | |
| runs-on: ubuntu-latest | |
| title: AMD64 Debian C++23 | |
| env: | |
| ARCHERY_DEBUG: 1 | |
| ARROW_ENABLE_TIMING_TESTS: OFF | |
| DOCKER_VOLUME_PREFIX: ".docker/" | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Cache Docker Volumes | |
| uses: actions/cache@v5 | |
| with: | |
| path: .docker | |
| key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} | |
| restore-keys: extra-${{ matrix.image }}- | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: 3 | |
| - name: Setup Archery | |
| run: python3 -m pip install -e dev/archery[docker] | |
| - name: Execute Docker Build | |
| continue-on-error: ${{ matrix.continue-on-error || false }} | |
| env: | |
| ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} | |
| ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} | |
| ENVS: ${{ toJSON(matrix.envs) }} | |
| run: | | |
| # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes | |
| sudo sysctl -w vm.mmap_rnd_bits=28 | |
| source ci/scripts/util_enable_core_dumps.sh | |
| if [ "${ENVS}" != "null" ]; then | |
| echo "${ENVS}" | jq -r '.[]' | while read env; do | |
| echo "${env}" >> .env | |
| done | |
| fi | |
| archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} | |
| - name: Docker Push | |
| if: >- | |
| success() && | |
| github.event_name == 'push' && | |
| github.repository == 'apache/arrow' && | |
| github.ref_name == 'main' | |
| env: | |
| ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} | |
| ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} | |
| continue-on-error: true | |
| run: archery docker push ${{ matrix.image }} | |
| msvc-arm64: | |
| needs: check-labels | |
| if: >- | |
| needs.check-labels.outputs.force == 'true' || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') | |
| name: ARM64 Windows 11 MSVC | |
| uses: ./.github/workflows/cpp_windows.yml | |
| with: | |
| arch: arm64 | |
| os: windows-11-arm | |
| simd-level: NONE | |
| jni-linux: | |
| needs: check-labels | |
| name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} | |
| runs-on: ${{ matrix.platform.runs-on }} | |
| if: >- | |
| needs.check-labels.outputs.force == 'true' || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') | |
| timeout-minutes: 240 | |
| permissions: | |
| # This is for using GitHub Packages for vcpkg cache | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - arch: "amd64" | |
| runs-on: ubuntu-latest | |
| - arch: "arm64v8" | |
| runs-on: ubuntu-24.04-arm | |
| env: | |
| ARCH: ${{ matrix.platform.arch }} | |
| REPO: ghcr.io/${{ github.repository }}-dev | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Free up disk space | |
| run: | | |
| ci/scripts/util_free_space.sh | |
| - name: Cache Docker Volumes | |
| uses: actions/cache@v5 | |
| with: | |
| path: .docker | |
| key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} | |
| restore-keys: jni-${{ matrix.platform.runs-on }}- | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: 3 | |
| - name: Setup Archery | |
| run: python3 -m pip install -e dev/archery[docker] | |
| - name: Execute Docker Build | |
| env: | |
| ARCHERY_DOCKER_USER: ${{ github.actor }} | |
| ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" | |
| run: | | |
| source ci/scripts/util_enable_core_dumps.sh | |
| archery docker run cpp-jni | |
| - name: Docker Push | |
| if: >- | |
| success() && | |
| github.event_name == 'push' && | |
| github.ref_name == 'main' | |
| env: | |
| ARCHERY_DOCKER_USER: ${{ github.actor }} | |
| ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| continue-on-error: true | |
| run: archery docker push cpp-jni | |
| jni-macos: | |
| needs: check-labels | |
| name: JNI macOS | |
| runs-on: macos-14 | |
| if: >- | |
| needs.check-labels.outputs.force == 'true' || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') | |
| timeout-minutes: 45 | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: "14.0" | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Install dependencies | |
| run: | | |
| brew bundle --file=cpp/Brewfile | |
| # We want to link aws-sdk-cpp statically but Homebrew's | |
| # aws-sdk-cpp provides only shared library. If we have | |
| # Homebrew's aws-sdk-cpp, our build mix Homebrew's | |
| # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's | |
| # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. | |
| brew uninstall aws-sdk-cpp | |
| # We want to use bundled RE2 for static linking. If | |
| # Homebrew's RE2 is installed, its header file may be used. | |
| # We uninstall Homebrew's RE2 to ensure using bundled RE2. | |
| brew uninstall grpc || : # gRPC depends on RE2 | |
| brew uninstall [email protected] || : # gRPC 1.54 may be installed too | |
| brew uninstall re2 | |
| # We want to use bundled Protobuf for static linking. If | |
| # Homebrew's Protobuf is installed, its library file may be | |
| # used on test We uninstall Homebrew's Protobuf to ensure using | |
| # bundled Protobuf. | |
| brew uninstall protobuf | |
| - name: Prepare ccache | |
| run: | | |
| echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} | |
| - name: Cache ccache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ccache | |
| key: jni-macos-${{ hashFiles('cpp/**') }} | |
| restore-keys: jni-macos- | |
| - name: CMake | |
| run: | | |
| cmake \ | |
| -S cpp \ | |
| -B cpp.build \ | |
| --preset=ninja-release-jni-macos \ | |
| -DARROW_BUILD_TESTS=ON \ | |
| -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install | |
| - name: Build | |
| run: | | |
| cmake --build cpp.build | |
| - name: Install | |
| run: | | |
| cmake --install cpp.build | |
| - name: Test | |
| env: | |
| ARROW_TEST_DATA: ${{ github.workspace }}/testing/data | |
| PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data | |
| run: | | |
| # MinIO is required | |
| exclude_tests="arrow-s3fs-test" | |
| # unstable | |
| exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" | |
| exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" | |
| ctest \ | |
| --exclude-regex "${exclude_tests}" \ | |
| --label-regex unittest \ | |
| --output-on-failure \ | |
| --parallel "$(sysctl -n hw.ncpu)" \ | |
| --test-dir "cpp.build" \ | |
| --timeout 300 | |
| - name: Build example | |
| run: | | |
| cmake \ | |
| -S cpp/examples/minimal_build/ \ | |
| -B cpp/examples/minimal_build.build \ | |
| -GNinja \ | |
| -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install | |
| cmake --build cpp/examples/minimal_build.build | |
| cd cpp/examples/minimal_build | |
| ../minimal_build.build/arrow-example | |
| odbc: | |
| needs: check-labels | |
| name: ODBC | |
| runs-on: windows-2022 | |
| if: >- | |
| needs.check-labels.outputs.force == 'true' || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || | |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') | |
| timeout-minutes: 240 | |
| permissions: | |
| packages: write | |
| env: | |
| ARROW_BUILD_SHARED: ON | |
| ARROW_BUILD_STATIC: OFF | |
| ARROW_BUILD_TESTS: ON | |
| ARROW_BUILD_TYPE: release | |
| ARROW_DEPENDENCY_SOURCE: VCPKG | |
| ARROW_FLIGHT_SQL_ODBC: ON | |
| ARROW_HOME: /usr | |
| ARROW_SIMD_LEVEL: AVX2 | |
| CMAKE_GENERATOR: Ninja | |
| CMAKE_INSTALL_PREFIX: /usr | |
| VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' | |
| VCPKG_DEFAULT_TRIPLET: x64-windows | |
| steps: | |
| - name: Disable Crash Dialogs | |
| run: | | |
| reg add ` | |
| "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` | |
| /v DontShowUI ` | |
| /t REG_DWORD ` | |
| /d 1 ` | |
| /f | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Download Timezone Database | |
| shell: bash | |
| run: ci/scripts/download_tz_database.sh | |
| - name: Install cmake | |
| shell: bash | |
| run: | | |
| ci/scripts/install_cmake.sh 4.1.2 /usr | |
| - name: Install ccache | |
| shell: bash | |
| run: | | |
| ci/scripts/install_ccache.sh 4.12.1 /usr | |
| - name: Setup ccache | |
| shell: bash | |
| run: | | |
| ci/scripts/ccache_setup.sh | |
| - name: ccache info | |
| id: ccache-info | |
| shell: bash | |
| run: | | |
| echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT | |
| - name: Cache ccache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ steps.ccache-info.outputs.cache-dir }} | |
| key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }} | |
| restore-keys: cpp-odbc-ccache-windows-x64- | |
| - name: Checkout vcpkg | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| path: vcpkg | |
| repository: microsoft/vcpkg | |
| - name: Bootstrap vcpkg | |
| run: | | |
| vcpkg\bootstrap-vcpkg.bat | |
| $VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString() | |
| Write-Output ${VCPKG_ROOT} | ` | |
| Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append | |
| Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | ` | |
| Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append | |
| - name: Setup NuGet credentials for vcpkg caching | |
| shell: bash | |
| run: | | |
| $(vcpkg fetch nuget | tail -n 1) \ | |
| sources add \ | |
| -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \ | |
| -storepasswordincleartext \ | |
| -name "GitHub" \ | |
| -username "$GITHUB_REPOSITORY_OWNER" \ | |
| -password "${{ secrets.GITHUB_TOKEN }}" | |
| $(vcpkg fetch nuget | tail -n 1) \ | |
| setapikey "${{ secrets.GITHUB_TOKEN }}" \ | |
| -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" | |
| - name: Show vcpkg NuGet pwsh | |
| shell: pwsh | |
| run: | | |
| vcpkg fetch nuget | |
| - name: Show vcpkg NuGet bash | |
| shell: bash | |
| run: | | |
| vcpkg fetch nuget | |
| # -AL- temp disable. | |
| - name: Build | |
| shell: cmd | |
| run: | | |
| set VCPKG_ROOT_KEEP=%VCPKG_ROOT% | |
| call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 | |
| set VCPKG_ROOT=%VCPKG_ROOT_KEEP% | |
| vcpkg install abseil --debug | |
| bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" | |
| - name: Register Flight SQL ODBC Driver | |
| shell: cmd | |
| run: | | |
| call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\arrow_flight_sql_odbc.dll | |
| # GH-48270 TODO: Resolve segementation fault during Arrow library unload | |
| # GH-48269 TODO: Enable Flight & Flight SQL testing in MSVC CI | |
| # GH-48547 TODO: enable ODBC tests after GH-48270 and GH-48269 are resolved. | |
| # GH-47787 TODO: Build ODBC installer | |
| report-extra-cpp: | |
| if: github.event_name == 'schedule' && always() | |
| needs: | |
| - docker | |
| - jni-linux | |
| - jni-macos | |
| - msvc-arm64 | |
| - odbc | |
| uses: ./.github/workflows/report_ci.yml | |
| secrets: inherit |