CMake: Stop building tests by default (related to #20539) (#24373)

#test-continuous

Related to #20539, in reaction to #20539 (comment)

CC @mkruskal-google

Closes #24373

COPYBARA_INTEGRATE_REVIEW=#24373 from hartwork:cmake-stop-building-tests-by-default 6be6748
FUTURE_COPYBARA_INTEGRATE_REVIEW=#24373 from hartwork:cmake-stop-building-tests-by-default 6be6748
PiperOrigin-RevId: 831563407

Author: hartwork

.github/scripts/validate_yaml.py

@@ -41,6 +41,23 @@
 
   print('PASSED: All jobs are in the list of blocking jobs.')
 
+
+def check_ctest(name, job, file, obj):
+  """Checks that the ctest command is not in a mode that allows missing tests to pass."""
+  for key in obj:
+    if (
+        isinstance(obj[key], str)
+        and 'ctest' in obj[key]
+        and 'ctest --no-tests=error' not in obj[key]
+    ):
+      raise ValueError(
+          'Step %s in job %s from file %s runs ctest in a mode that'
+          ' allows missing tests to pass' % (name, job, file)
+      )
+    elif isinstance(obj[key], dict):
+      check_ctest(name, job, file, obj[key])
+
+
 # Ensure every job with a continuous prefix conditions every step on whether we
 # are in a continuous run.
 for file in yaml_files:
@@ -72,4 +89,6 @@
               ' condition but the job does not contain the continuous-prefix'
               % (name, job, file)
           )
+        check_ctest(name, job, file, step)
+
 print('PASSED: All steps in all jobs check the continuous-run condition.')

.github/workflows/test_cpp.yml

@@ -146,10 +146,11 @@ jobs:
             -c "set -ex;
             sccache -z;
             cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
+            -Dprotobuf_BUILD_TESTS=ON
             -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=17
             -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
             cmake --build . --parallel 20;
-            ctest --parallel 20;
+            ctest --no-tests=error --parallel 20;
             sccache -s"
 
   linux-cmake:
@@ -191,7 +192,7 @@ jobs:
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         uses: protocolbuffers/protobuf-ci/docker@v5
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-e0df73e51131ccaf53451355d22577f377357604
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-9626718698895971df3953d4aa2321d7425f3c5f
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }}
@@ -234,7 +235,7 @@ jobs:
         uses: protocolbuffers/protobuf-ci/docker@v5
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-e0df73e51131ccaf53451355d22577f377357604
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-9626718698895971df3953d4aa2321d7425f3c5f
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /install.sh -DCMAKE_CXX_STANDARD=17 ${{ env.SCCACHE_CMAKE_FLAGS }}
@@ -332,12 +333,12 @@ jobs:
         if: ${{ inputs.continuous-run }}
         uses: protocolbuffers/protobuf-ci/docker@v5
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-e0df73e51131ccaf53451355d22577f377357604
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-9626718698895971df3953d4aa2321d7425f3c5f
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /install.sh -DCMAKE_CXX_STANDARD=17 ${{ env.SCCACHE_CMAKE_FLAGS }}
             -Dprotobuf_LOCAL_DEPENDENCIES_ONLY=OFF
-            -Dprotobuf_BUILD_EXAMPLES=OFF \&\&
+            -Dprotobuf_BUILD_EXAMPLES=OFF -Dprotobuf_BUILD_TESTS=ON \&\&
             mkdir examples/build \&\&
             cd examples/build \&\&
             cmake .. -DCMAKE_CXX_STANDARD=17 \&\&
@@ -381,9 +382,9 @@ jobs:
               -c 'set -ex;
               cd /workspace;
               sccache -z;
-              cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }};
+              cmake . -Dprotobuf_BUILD_TESTS=ON ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }};
               cmake --build . --parallel 20;
-              ctest --verbose --parallel 20;
+              ctest --no-tests=error --verbose --parallel 20;
               sccache -s'
 
   linux-cmake-32-bit:
@@ -407,13 +408,14 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:d6028ab408c49932836cdc514116f06886d7f6868a4d430630aa52adc5aee2fc
           platform: linux/386
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          entrypoint: bash
           command: >-
-              /bin/bash -cex '
+              -c 'set -ex;
               cd /workspace;
               sccache -z;
-              cmake . -DCMAKE_CXX_STANDARD=17 ${{ env.SCCACHE_CMAKE_FLAGS }};
+              cmake . -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
               cmake --build . --parallel 20;
-              ctest --verbose --parallel 20;
+              ctest --no-tests=error --verbose --parallel 20;
               sccache -s'
 
   non-linux:
@@ -530,7 +532,7 @@ jobs:
           bazel-version: 8.0.1
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
-            cmake . -DCMAKE_CXX_STANDARD=17 ${{ matrix.install-flags }}
+            cmake . -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=ON ${{ matrix.install-flags }}
             ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
       - name: Build for install
         if: ${{ matrix.install-flags && (!matrix.continuous-only || inputs.continuous-run) }}
@@ -556,7 +558,7 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-version: 8.0.1
           command: >-
-            cmake . -DCMAKE_CXX_STANDARD=17 ${{ matrix.flags }}
+            cmake . -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=ON ${{ matrix.flags }}
             ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
 
       - name: Build
@@ -567,7 +569,7 @@ jobs:
       - name: Test
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         shell: bash
-        run: ctest --verbose --parallel 20 -C Debug
+        run: ctest --no-tests=error --verbose --parallel 20 -C Debug
 
       - name: Report sccache stats
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}

CMakeLists.txt

@@ -29,7 +29,7 @@ endif()
 
 # Options
 option(protobuf_INSTALL "Install protobuf binaries and files" ON)
-option(protobuf_BUILD_TESTS "Build tests" ON)
+option(protobuf_BUILD_TESTS "Build tests" OFF)
 option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF)
 option(protobuf_BUILD_EXAMPLES "Build examples" OFF)
 option(protobuf_BUILD_PROTOBUF_BINARIES "Build protobuf libraries and protoc compiler" ON)

appveyor.bat

@@ -12,7 +12,7 @@ echo Building MinGW
 set PATH=C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH:C:\Program Files\Git\usr\bin;=%
 mkdir build_mingw
 cd build_mingw
-cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=0 ../cmake
+cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% ../cmake
 mingw32-make -j8 all || goto error
 rem cd %configuration%
 rem tests.exe || goto error

cmake/README.md

@@ -89,11 +89,11 @@ downloading and building a copy from GitHub. To prevent this and make it an
 error condition, you can optionally set:
 `-Dprotobuf_LOCAL_DEPENDENCIES_ONLY=ON`.
 
-### Disabling Tests
+### Enabling Tests
 
-To disable building the unit tests completely, set the following flag:
+To enable building the unit tests, set the following flag:
 
-*   `-Dprotobuf_BUILD_TESTS=OFF`
+*   `-Dprotobuf_BUILD_TESTS=ON`
 
 ### ZLib Support
 

cmake/gtest.cmake

@@ -22,7 +22,5 @@ endif()
 
 if (NOT TARGET GTest::gmock)
   message(FATAL_ERROR
-          "Cannot find googletest dependency that's needed to build tests.\n"
-          "If instead you want to skip tests, run cmake with:\n"
-          "  cmake -Dprotobuf_BUILD_TESTS=OFF\n")
+          "Cannot find googletest dependency that's needed to build tests.\n")
 endif()