[release-22.0] Generate race unit tests (#19078) (#19295)

Signed-off-by: Mohamed Hamza <mhamza@fastmail.com>
Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>
Co-authored-by: Mohamed Hamza <mhamza@fastmail.com>

Author: vitess-bot[bot]

.github/workflows/unit_race.yml

@@ -1,4 +1,6 @@
-name: unit_race
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Unit Test (Race)
 on:
   push:
     branches:
@@ -8,7 +10,7 @@ on:
   pull_request:
     branches: '**'
 concurrency:
-  group: format('{0}-{1}', ${{ github.ref }}, 'unit_race')
+  group: format('{0}-{1}', ${{ github.ref }}, 'Unit Test (Race)')
   cancel-in-progress: true
 
 permissions: read-all
@@ -19,10 +21,10 @@ env:
   GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"
 
 jobs:
-
-  build:
+  test:
     name: Unit Test (Race)
     runs-on: gh-hosted-runners-16cores-1-24.04
+
     steps:
     - name: Skip CI
       run: |
@@ -75,22 +77,20 @@ jobs:
       uses: ./.github/actions/setup-mysql
       with:
         flavor: mysql-8.0
-
+        
     - name: Get dependencies
       if: steps.changes.outputs.unit_tests == 'true'
       run: |
         export DEBIAN_FRONTEND="noninteractive"
-        sudo apt-get -qq update
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk
 
-        sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk
-        
         mkdir -p dist bin
         curl --max-time 10 --retry 3 --retry-max-time 45 -s -L https://github.com/coreos/etcd/releases/download/v3.5.25/etcd-v3.5.25-linux-amd64.tar.gz | tar -zxC dist
         mv dist/etcd-v3.5.25-linux-amd64/{etcd,etcdctl} bin/
 
         go mod download
         go install golang.org/x/tools/cmd/goimports@latest
-        
+
         # install JUnit report formatter
         go install github.com/vitessio/go-junit-report@HEAD
 
@@ -111,15 +111,20 @@ jobs:
         # Tell Launchable about the build you are producing and testing
         launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source .
 
-    - name: unit_race
+    - name: Run test
       if: steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 45
       run: |
+        set -exo pipefail
         # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
         # which musn't be more than 107 characters long.
         export VTDATAROOT="/tmp/"
+
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
 
         make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
@@ -134,7 +139,7 @@ jobs:
       run: |
         # print test output
         cat output.txt
-        
+
     - name: Test Summary
       if: steps.changes.outputs.unit_tests == 'true' && !cancelled()
       uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4

.github/workflows/unit_race_evalengine.yml

@@ -1,4 +1,6 @@
-name: unit_race_evalengine
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Unit Test (Evalengine_Race)
 on:
   push:
     branches:
@@ -8,7 +10,7 @@ on:
   pull_request:
     branches: '**'
 concurrency:
-  group: format('{0}-{1}', ${{ github.ref }}, 'unit_race_evalengine')
+  group: format('{0}-{1}', ${{ github.ref }}, 'Unit Test (Evalengine_Race)')
   cancel-in-progress: true
 
 permissions: read-all
@@ -19,10 +21,10 @@ env:
   GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"
 
 jobs:
-
-  build:
+  test:
     name: Unit Test (Evalengine_Race)
     runs-on: gh-hosted-runners-16cores-1-24.04
+
     steps:
     - name: Skip CI
       run: |
@@ -75,22 +77,20 @@ jobs:
       uses: ./.github/actions/setup-mysql
       with:
         flavor: mysql-8.0
-
+        
     - name: Get dependencies
       if: steps.changes.outputs.unit_tests == 'true'
       run: |
         export DEBIAN_FRONTEND="noninteractive"
-        sudo apt-get -qq update
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk
 
-        sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk
-        
         mkdir -p dist bin
         curl --max-time 10 --retry 3 --retry-max-time 45 -s -L https://github.com/coreos/etcd/releases/download/v3.5.25/etcd-v3.5.25-linux-amd64.tar.gz | tar -zxC dist
         mv dist/etcd-v3.5.25-linux-amd64/{etcd,etcdctl} bin/
 
         go mod download
         go install golang.org/x/tools/cmd/goimports@latest
-        
+
         # install JUnit report formatter
         go install github.com/vitessio/go-junit-report@HEAD
 
@@ -111,15 +111,20 @@ jobs:
         # Tell Launchable about the build you are producing and testing
         launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source .
 
-    - name: unit_race_evalengine
+    - name: Run test
       if: steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 45
       run: |
+        set -exo pipefail
         # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
         # which musn't be more than 107 characters long.
         export VTDATAROOT="/tmp/"
+
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
 
         make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

Makefile

@@ -228,7 +228,7 @@ unit_test_cover: build dependency_check demo
 	go tool $(VT_GO_PARALLEL) cover -html=coverage.out
 
 unit_test_race: build dependency_check
-	tools/unit_test_race.sh
+	RACE=1 tools/unit_test_runner.sh
 
 e2e_test_race: build
 	tools/e2e_test_race.sh

go/vt/vtctl/workflow/utils_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/require"
 	clientv3 "go.etcd.io/etcd/client/v3"
 
+	"vitess.io/vitess/go/race"
 	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/testfiles"
 	"vitess.io/vitess/go/vt/log"
@@ -119,10 +120,10 @@ func TestUpdateKeyspaceRoutingRule(t *testing.T) {
 // TestConcurrentKeyspaceRoutingRulesUpdates runs multiple keyspace routing rules updates concurrently to test
 // the locking mechanism.
 func TestConcurrentKeyspaceRoutingRulesUpdates(t *testing.T) {
-	if os.Getenv("GOCOVERDIR") != "" {
-		// While running this test in CI along with all other tests in for code coverage this test hangs very often.
-		// Possibly due to some resource constraints, since this test is one of the last.
-		// However just running this package by itself with code coverage works fine in CI.
+	if os.Getenv("GOCOVERDIR") != "" || race.Enabled {
+		// While running this test in CI along with all other tests in for code coverage or race enabled,
+		// this test hangs very often. Possibly due to some resource constraints, since this test is one
+		// of the last. However just running this package by itself with code coverage works fine in CI.
 		t.Logf("Skipping TestConcurrentKeyspaceRoutingRulesUpdates test in code coverage mode")
 		t.Skip()
 	}

test/ci_workflow_gen.go

@@ -38,13 +38,9 @@ const (
 
 type mysqlVersions []mysqlVersion
 
-var (
-	defaultMySQLVersions = []mysqlVersion{defaultMySQLVersion}
-)
+var defaultMySQLVersions = []mysqlVersion{defaultMySQLVersion}
 
-var (
-	unitTestDatabases = []mysqlVersion{mysql57, mysql80, mysql84}
-)
+var unitTestDatabases = []mysqlVersion{mysql57, mysql80, mysql84}
 
 const (
 	oracleCloudRunner = "oracle-16cpu-64gb-x86-64"
@@ -180,6 +176,7 @@ var (
 
 type unitTest struct {
 	Name, RunsOn, Platform, FileName, GoPrivate, Evalengine string
+	Race                                                    bool
 }
 
 type clusterTest struct {
@@ -374,6 +371,24 @@ func generateUnitTestWorkflows() {
 			}
 		}
 	}
+
+	// Generate unit tests with race detection
+	for _, evalengine := range []string{"1", "0"} {
+		raceTest := &unitTest{
+			Name:       fmt.Sprintf("Unit Test (%sRace)", evalengineToRaceNamePrefix(evalengine)),
+			RunsOn:     cores16RunnerName,
+			Platform:   string(mysql80),
+			GoPrivate:  goPrivate,
+			Evalengine: evalengine,
+			Race:       true,
+		}
+		raceTest.FileName = fmt.Sprintf("unit_race%s.yml", evalengineToFileSuffix(evalengine))
+		path := fmt.Sprintf("%s/%s", workflowConfigDir, raceTest.FileName)
+		err := writeFileFromTemplate(unitTestTemplate, path, raceTest)
+		if err != nil {
+			log.Print(err)
+		}
+	}
 }
 
 func evalengineToString(evalengine string) string {
@@ -383,6 +398,27 @@ func evalengineToString(evalengine string) string {
 	return ""
 }
 
+func evalengineToNameSuffix(evalengine string) string {
+	if evalengine == "1" {
+		return " evalengine"
+	}
+	return ""
+}
+
+func evalengineToRaceNamePrefix(evalengine string) string {
+	if evalengine == "1" {
+		return "Evalengine_"
+	}
+	return ""
+}
+
+func evalengineToFileSuffix(evalengine string) string {
+	if evalengine == "1" {
+		return "_evalengine"
+	}
+	return ""
+}
+
 func writeFileFromTemplate(templateFile, filePath string, test any) error {
 	tpl := template.New(path.Base(templateFile))
 	tpl.Funcs(template.FuncMap{

test/templates/unit_test.tpl

@@ -126,7 +126,7 @@ jobs:
 
     - name: Run test
       if: steps.changes.outputs.unit_tests == 'true'
-      timeout-minutes: 30
+      timeout-minutes: {{if .Race}}45{{else}}30{{end}}
       run: |
         set -exo pipefail
         # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
@@ -139,7 +139,7 @@ jobs:
         # testing, e.g. MySQL 5.7 vs 8.0.
         export CI_DB_PLATFORM="{{.Platform}}"
 
-        make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
+        make {{if .Race}}unit_test_race{{else}}unit_test{{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
     - name: Record test results in launchable if PR is not a draft
       if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled()

tools/unit_test_race.sh

@@ -36,6 +36,12 @@ if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then
   VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE"
 fi
 
+# Enable race detector if RACE=1
+RACE_FLAG=""
+if [[ "$RACE" == "1" ]]; then
+  RACE_FLAG="-race"
+fi
+
 # Mac makes long temp directories for os.TempDir(). MySQL can't connect to
 # sockets in those directories. Tell Golang to use /tmp/vttest_XXXXXX instead.
 kernel="$(uname -s)"
@@ -68,7 +74,7 @@ all_except_flaky_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_tes
 flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | cut -d" " -f1)
 
 # Run non-flaky tests.
-echo "$all_except_flaky_tests" | xargs go test $VT_GO_PARALLEL -v -count=1
+go test $VT_GO_PARALLEL $RACE_FLAG -v -count=1 $all_except_flaky_tests
 if [ $? -ne 0 ]; then
   echo "ERROR: Go unit tests failed. See above for errors."
   echo
@@ -84,7 +90,7 @@ for pkg in $flaky_tests; do
   max_attempts=3
   attempt=1
   # Set a timeout because some tests may deadlock when they flake.
-  until go test -timeout 5m $VT_GO_PARALLEL $pkg -v -count=1; do
+  until go test -timeout 5m $VT_GO_PARALLEL $RACE_FLAG $pkg -v -count=1; do
     echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors."
     if [ $((++attempt)) -gt $max_attempts ]; then
       echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness."