VTOrc: detect stalled replication on a full disk

.github/workflows/cluster_endtoend.yml

@@ -46,7 +46,8 @@ jobs:
             "docker_cluster",
             "5",
             "28",
-            "onlineddl_flow"
+            "onlineddl_flow",
+            "vtorc_disk_full"
           ]'
 
           # Build matrix

.github/workflows/vtorc_disk_full_e2e.yml

@@ -0,0 +1,102 @@
+name: VTOrc Disk-Full E2E
+
+on:
+  push:
+    branches:
+      - "main"
+      - "release-[0-9]+.[0-9]"
+    tags: "**"
+  pull_request:
+    branches: "**"
+
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'VTOrc Disk-Full E2E')
+  cancel-in-progress: true
+
+permissions: read-all
+
+jobs:
+  test:
+    name: VTOrc replication-stalled disk-full
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: "false"
+
+      - name: Check for changes in relevant files
+        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: changes
+        with:
+          token: ""
+          filters: |
+            relevant:
+              - 'go/**/*.go'
+              - 'config/**'
+              - 'proto/*.proto'
+              - 'test/config.json'
+              - 'go.mod'
+              - 'go.sum'
+              - 'Makefile'
+              - 'build.env'
+              - 'bootstrap.sh'
+              - '.github/workflows/vtorc_disk_full_e2e.yml'
+
+      - name: Set up Go
+        if: steps.changes.outputs.relevant == 'true'
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version-file: go.mod
+          cache: ${{ (github.base_ref == 'main' || (github.base_ref == '' && github.ref_name == 'main')) && 'true' || 'false' }}
+
+      - name: Tune the OS
+        if: steps.changes.outputs.relevant == 'true'
+        timeout-minutes: 5
+        uses: ./.github/actions/tune-os
+
+      - name: Setup MySQL
+        if: steps.changes.outputs.relevant == 'true'
+        timeout-minutes: 8
+        uses: ./.github/actions/setup-mysql
+        with:
+          flavor: mysql-8.4
+
+      - name: Get dependencies
+        if: steps.changes.outputs.relevant == 'true'
+        timeout-minutes: 10
+        run: |
+          # e2fsprogs ships mkfs.ext4; mount/util-linux is preinstalled.
+          sudo apt-get -qq update
+          sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 e2fsprogs
+          sudo service etcd stop
+          go mod download
+
+      - name: Build Vitess binaries
+        if: steps.changes.outputs.relevant == 'true'
+        timeout-minutes: 10
+        run: |
+          source build.env
+          make build
+
+      - name: Run replication-stalled disk-full e2e
+        if: steps.changes.outputs.relevant == 'true'
+        timeout-minutes: 20
+        run: |
+          # The test mounts a loopback ext4 filesystem and runs mkfs.ext4 on
+          # it; both require root. VT_TEST_DISK_FULL=1 is the explicit
+          # opt-in checked by the test's TestMain (in addition to the
+          # //go:build linux gate that already prevents it from compiling on
+          # macOS).
+          export VTDATAROOT="/tmp/vtdataroot_diskfull"
+          mkdir -p "$VTDATAROOT"
+          source build.env
+          set -exo pipefail
+
+          # build.env prepended $PWD/bin to PATH; preserve via sudo -E.
+          # VT_TEST_DISK_FULL=1 is the explicit opt-in checked by the test.
+          sudo -E env VT_TEST_DISK_FULL=1 PATH="$PATH" \
+            go test -timeout 15m -count=1 -v \
+            ./go/test/endtoend/vtorc/replicationstalleddiskfull/...

go/mysql/capabilities/capability.go

@@ -51,6 +51,7 @@ const (
 	BinaryLogStatus                                                       // Supported in 8.2.0 and above, uses SHOW BINARY LOG STATUS
 	RestrictFKOnNonStandardKey                                            // Supported in 8.4.0 and above, restricts usage of non-standard indexes for foreign keys.
 	MySQLClonePluginFlavorCapability                                      // Supported in 8.0.17 and above, MySQL CLONE plugin for physical snapshot.
+	PerformanceSchemaErrorLogTableCapability                              // Supported in 8.0.22 and above: performance_schema.error_log table.
 )
 
 type CapableOf func(capability FlavorCapability) (bool, error)
@@ -111,6 +112,8 @@ func MySQLVersionHasCapability(serverVersion string, capability FlavorCapability
 		return atLeast(8, 0, 17)
 	case DisableRedoLogFlavorCapability:
 		return atLeast(8, 0, 21)
+	case PerformanceSchemaErrorLogTableCapability:
+		return atLeast(8, 0, 22)
 	case FastDropTableFlavorCapability:
 		return atLeast(8, 0, 23)
 	case InstantChangeColumnVisibilityCapability:

go/test/endtoend/cluster/cluster_process.go

@@ -478,6 +478,13 @@ func (cluster *LocalProcessCluster) AddShard(keyspaceName string, shardName stri
 		}
 
 		tablet.MysqlctlProcess = *mysqlctlProcess
+		// Apply mysqlctl customizations before StartProcess so per-tablet
+		// settings (e.g. ExtraMyCnfPath) take effect on the spawned mysqld.
+		for _, customizer := range customizers {
+			if f, ok := customizer.(func(*MysqlctlProcess)); ok {
+				f(&tablet.MysqlctlProcess)
+			}
+		}
 		proc, err := tablet.MysqlctlProcess.StartProcess()
 		if err != nil {
 			log.Error(fmt.Sprintf("error starting mysqlctl process: %v, %v", tablet.MysqlctldProcess, err))
@@ -513,9 +520,12 @@ func (cluster *LocalProcessCluster) AddShard(keyspaceName string, shardName stri
 		shard.Vttablets = append(shard.Vttablets, tablet)
 		// Apply customizations
 		for _, customizer := range customizers {
-			if f, ok := customizer.(func(*VttabletProcess)); ok {
+			switch f := customizer.(type) {
+			case func(*VttabletProcess):
 				f(tablet.VttabletProcess)
-			} else {
+			case func(*MysqlctlProcess):
+				// Already applied above, before StartProcess.
+			default:
 				return nil, fmt.Errorf("type mismatch on customizer: %T", customizer)
 			}
 		}
@@ -599,6 +609,11 @@ func (cluster *LocalProcessCluster) StartKeyspaceLegacy(keyspace Keyspace, shard
 				return err
 			}
 			tablet.MysqlctlProcess = *mysqlctlProcess
+			for _, customizer := range customizers {
+				if f, ok := customizer.(func(*MysqlctlProcess)); ok {
+					f(&tablet.MysqlctlProcess)
+				}
+			}
 			proc, err := tablet.MysqlctlProcess.StartProcess()
 			if err != nil {
 				log.Error(fmt.Sprintf("error starting mysqlctl process: %v, %v", tablet.MysqlctldProcess, err))
@@ -628,9 +643,12 @@ func (cluster *LocalProcessCluster) StartKeyspaceLegacy(keyspace Keyspace, shard
 			shard.Vttablets = append(shard.Vttablets, tablet)
 			// Apply customizations
 			for _, customizer := range customizers {
-				if f, ok := customizer.(func(*VttabletProcess)); ok {
+				switch f := customizer.(type) {
+				case func(*VttabletProcess):
 					f(tablet.VttabletProcess)
-				} else {
+				case func(*MysqlctlProcess):
+					// Already applied above, before StartProcess.
+				default:
 					return fmt.Errorf("type mismatch on customizer: %T", customizer)
 				}
 			}

go/test/endtoend/cluster/mysqlctl_process.go

@@ -48,6 +48,10 @@ type MysqlctlProcess struct {
 	ExtraArgs       []string
 	InitMysql       bool
 	SecureTransport bool
+	// ExtraMyCnfPath, when set, is appended to EXTRA_MY_CNF on the per-process
+	// env so its directives are loaded after the generated cnf and override it.
+	// Only this mysqlctl invocation sees it; other tablets' env is untouched.
+	ExtraMyCnfPath string
 }
 
 // InitDb executes mysqlctl command to add cell info
@@ -109,6 +113,7 @@ func (mysqlctl *MysqlctlProcess) startProcess(init bool) (*exec.Cmd, error) {
 	if len(mysqlctl.ExtraArgs) > 0 {
 		tmpProcess.Args = append(tmpProcess.Args, mysqlctl.ExtraArgs...)
 	}
+	var extraCnfPaths []string
 	if mysqlctl.InitMysql {
 		if mysqlctl.SecureTransport {
 			// Set up EXTRA_MY_CNF for ssl
@@ -145,7 +150,7 @@ ssl_key={{.ServerKey}}
 				return nil, err
 			}
 
-			tmpProcess.Env = append(tmpProcess.Env, "EXTRA_MY_CNF="+extraMyCNF)
+			extraCnfPaths = append(extraCnfPaths, extraMyCNF)
 			tmpProcess.Env = append(tmpProcess.Env, "VTDATAROOT="+os.Getenv("VTDATAROOT"))
 		}
 
@@ -158,6 +163,12 @@ ssl_key={{.ServerKey}}
 	} else {
 		tmpProcess.Args = append(tmpProcess.Args, "start")
 	}
+	if mysqlctl.ExtraMyCnfPath != "" {
+		extraCnfPaths = append(extraCnfPaths, mysqlctl.ExtraMyCnfPath)
+	}
+	if len(extraCnfPaths) > 0 {
+		tmpProcess.Env = append(tmpProcess.Env, "EXTRA_MY_CNF="+strings.Join(extraCnfPaths, ":"))
+	}
 	tmpProcess.Env = append(tmpProcess.Env, os.Environ()...)
 	tmpProcess.Env = append(tmpProcess.Env, DefaultVttestEnv)
 	log.Info(fmt.Sprintf("Starting mysqlctl with command: %v", tmpProcess.Args))