Update test regeneration script to support multiple Bazel versions

.bazelci/presubmit.yml

@@ -37,13 +37,13 @@ matrix:
   test_targets:
   # Some tests are expected to fail in legacy WORKSPACE mode due to repo name change
     - "//test/..."
-    - "-//test:attribute_defaults_test_e2e_test"
-    - "-//test:function_wrap_multiple_lines_test_e2e_test"
-    - "-//test:misc_apis_test_e2e_test"
-    - "-//test:module_extension_test_e2e_test"
-    - "-//test:proto_format_test_e2e_test"
+    - "-//test:attribute_defaults_test"
+    - "-//test:function_wrap_multiple_lines_test"
+    - "-//test:misc_apis_test"
+    - "-//test:module_extension_test"
+    - "-//test:proto_format_test"
     - "-//test:stardoc_self_gen_test"
-    - "-//test:table_of_contents_test_e2e_test"
+    - "-//test:table_of_contents_test"
 
 .windows_task_config: &windows_task_config
   <<: *common_task_config
@@ -90,15 +90,15 @@ tasks:
     platform: windows
     working_directory: test/bzlmod
 
-  bazel_7_1_tests:
-    name: Stardoc golden tests requiring Bazel 7.1
+  bazel_7_2_tests:
+    name: Stardoc golden tests requiring Bazel 7.2
     platform: ubuntu2004
-    bazel: 7.1.2
+    bazel: 7.2.0
     build_flags: *common_flags
     test_flags: *common_flags
     test_targets:
-    - "//test:proto_format_test_e2e_test"
-    - "//test:macro_kwargs_legacy_test_e2e_test"
+    - "//test:proto_format_test"
+    - "//test:macro_kwargs_legacy_test"
 
   bazel_8_tests:
     name: Stardoc golden tests requiring Bazel HEAD
@@ -107,6 +107,6 @@ tasks:
     build_flags: *common_flags
     test_flags: *common_flags
     test_targets:
-    - "//test:macro_kwargs_test_e2e_test"
+    - "//test:macro_kwargs_test"
 
 buildifier: latest

test/BUILD

@@ -44,9 +44,9 @@ stardoc_test(
     format = "proto",
     golden_file = "testdata/proto_format_test/golden.binaryproto",
     input_file = "testdata/proto_format_test/input.bzl",
-    # Golden output was generated with Bazel 7.1 and may differ in other versions
+    # Golden output was generated with Bazel 7.2 and may differ in other versions
     tags = [
-        "bazel_7_1",
+        "bazel_7_2",
         "manual",
     ],
 )
@@ -208,9 +208,9 @@ stardoc_test(
     name = "macro_kwargs_legacy_test",
     golden_file = "testdata/macro_kwargs_test/legacy_golden.md",
     input_file = "testdata/macro_kwargs_test/input.bzl",
-    # Golden output was generated with Bazel 7.1 and may differ in other versions
+    # Golden output was generated with Bazel 7.2 and may differ in other versions
     tags = [
-        "bazel_7_1",
+        "bazel_7_2",
         "manual",
     ],
 )

test/stardoc_test.bzl

@@ -29,11 +29,12 @@ def stardoc_test(
     Each invocation creates multiple targets:
 
     1. A `stardoc` target which will generate a new golden file given an input
-       file. This target should be used to regenerate the golden file when
-       updating Stardoc, named "regenerate_{name}_golden".
+       file, named "{name}_stardoc".
     2. An `sh_test` target which verifies that the output of the `stardoc`
        target above matches a golden file.
-    3. A bzl_library target for convenient wrapping of input bzl files, named "{name}_lib".
+    3. A shell script which can be executed via `bazel run` to update the golden
+       file from the `stardoc` target's output, named "{name}_regenerate"
+    4. A bzl_library target for convenient wrapping of input bzl files, named "{name}_lib".
 
     Args:
       name: A unique name to qualify the created targets.
@@ -52,8 +53,9 @@ def stardoc_test(
     )
 
     _create_test_targets(
-        test_name = "%s_e2e_test" % name,
-        genrule_name = "regenerate_%s_golden" % name,
+        test_name = name,
+        stardoc_name = "%s_stardoc" % name,
+        regenerate_name = "%s_regenerate" % name,
         lib_name = "%s_lib" % name,
         input_file = input_file,
         golden_file = golden_file,
@@ -63,13 +65,15 @@ def stardoc_test(
 
 def _create_test_targets(
         test_name,
-        genrule_name,
+        stardoc_name,
+        regenerate_name,
         lib_name,
         input_file,
         golden_file,
         test,
         **kwargs):
-    actual_generated_doc = "%s.out" % genrule_name
+    actual_generated_doc = "%s.md" % stardoc_name
+    tags = kwargs.get("tags", [])
 
     native.sh_test(
         name = test_name,
@@ -82,20 +86,40 @@ def _create_test_targets(
             actual_generated_doc,
             golden_file,
         ],
-        tags = kwargs.get("tags", []),
+        tags = tags,
+    )
+
+    regenerate_sh = "%s.sh" % regenerate_name
+    native.genrule(
+        name = "%s_sh" % regenerate_name,
+        cmd = """cat > $(location %s) <<EOF
+#!/usr/bin/env bash
+cd \\$${BUILD_WORKSPACE_DIRECTORY}
+cp -fv $(location %s) $(location %s)
+EOF""" % (regenerate_sh, actual_generated_doc, golden_file),
+        outs = [regenerate_sh],
+        srcs = [actual_generated_doc, golden_file],
+        tags = tags,
+    )
+
+    native.sh_binary(
+        name = regenerate_name,
+        srcs = [regenerate_sh],
+        data = [actual_generated_doc],
+        tags = tags,
     )
 
     if test == "default":
         stardoc(
-            name = genrule_name,
+            name = stardoc_name,
             out = actual_generated_doc,
             input = input_file,
             deps = [lib_name],
             **kwargs
         )
     elif test == "html_tables":
         html_tables_stardoc(
-            name = genrule_name,
+            name = stardoc_name,
             out = actual_generated_doc,
             input = input_file,
             deps = [lib_name],

update-stardoc-docs.sh

@@ -18,10 +18,11 @@
 set -eu
 
 # Allow users to override the bazel command with e.g. bazelisk.
-: "${BAZEL:=bazel}"
+: "${USE_BAZEL_VERSION:=8.0.0-pre.20240603.2}"
+: "${BAZEL:=bazelisk}"
 
 echo "** Generating Stardoc documentation..."
-${BAZEL} build //stardoc:stardoc_doc.md
+USE_BAZEL_VERSION="${USE_BAZEL_VERSION}" ${BAZEL} build //stardoc:stardoc_doc.md
 
 echo "** Copying result to docs/stardoc_rule.md ..."
 cp bazel-bin/stardoc/stardoc_doc.md docs/stardoc_rule.md

update-stardoc-tests.sh

@@ -32,32 +32,35 @@ function run_buildozer () {
   fi
 }
 
-# Allow users to override the bazel command with e.g. bazelisk.
-: "${BAZEL:=bazel}"
+function update_non_manual_tests () {
+  echo "** Querying for non-manual tests..."
+  regenerate $(${BAZEL} query "kind(sh_binary, //test:all) - attr(tags, manual, //test:all)" | grep _regenerate)
+}
 
-# Some tests cannot be automatically regenerated using this script, as they don't fall under the normal
-# golden test pattern or require a specific Bazel version
-EXCLUDED_TESTS="namespace_test_with_allowlist|multi_level_namespace_test_with_allowlist|local_repository_test|stamping_with_stamping_off|macro_kwargs"
-echo "** Querying for tests..."
-regen_targets=$(${BAZEL} query //test:all | grep regenerate_ | grep -vE "_golden\.extract|$EXCLUDED_TESTS")
+function update_manual_tests_with_tag () {
+  local manual_tag="$1"; shift
+  echo "** Querying for tests tagged \"${manual_tag}\", \"manual\" using 'USE_BAZEL_VERSION=${USE_BAZEL_VERSION} ${BAZEL}' ..."
+  regenerate $(${BAZEL} query "attr(tags, ${manual_tag}, attr(tags, manual, kind(sh_binary, //test:all)))" | grep _regenerate)
+}
 
-echo "** Building goldens..."
-${BAZEL} build $regen_targets
+function regenerate () {
+  echo "** Regenerating and copying goldens..."
+  for regen_target in $@; do
+    if [[ -z ${USE_BAZEL_VERSION+x} ]]; then
+      echo "** Running '${BAZEL} run ${regen_target}' ..."
+    else
+      echo "** Running 'USE_BAZEL_VERSION=${USE_BAZEL_VERSION} ${BAZEL} run ${regen_target}' ..."
+    fi
+    ${BAZEL} run "${regen_target}"
+  done
+}
 
-echo "** Copying goldens..."
-for regen_target in $regen_targets; do
-  base_target_name=$(echo $regen_target | sed 's/\/\/test://g')
-  testdata_pkg_name=$(echo $base_target_name | sed 's/regenerate_//g' | sed 's/_golden//g')
-  out_file="bazel-bin/test/${base_target_name}.out"
-  if [[ $regen_target == *"proto_format"* ]]; then
-    ext="binaryproto"
-  else
-    ext="md"
-  fi
-  golden="test/testdata/${testdata_pkg_name}/golden.${ext}"
-  cp "${out_file}" "${golden}"
-  chmod 644 "${golden}"
-done
+# Allow users to override the bazel command with e.g. bazelisk.
+: "${BAZEL:=bazelisk}"
+
+update_non_manual_tests
+USE_BAZEL_VERSION="7.2.0" update_manual_tests_with_tag "bazel_7_2"
+USE_BAZEL_VERSION="8.0.0-pre.20240603.2" update_manual_tests_with_tag "bazel_8"
 
 echo "** Files copied."
 echo "Please note that not all golden files are correctly copied by this script."