chunked builtin backup engine

changelog/25.0/25.0.0/summary.md

@@ -16,6 +16,8 @@
         - [Consolidator Reject on Waiter Cap](#vttablet-consolidator-reject-on-cap)
     - **[VTTablet](#minor-changes-vttablet)**
         - [Schema engine table-count limit is now configurable](#vttablet-schema-max-table-count)
+    - **[Backup/Restore](#minor-changes-backup)**
+        - [Chunked backup/restore for the builtinbackupengine](#backup-chunked-builtin)
 
 ## <a id="major-changes"/>Major Changes</a>
 
@@ -91,3 +93,18 @@ Two changes:
 Tablets that already have more tracked schema objects than the configured limit will reload fine — only new creations are gated. Operators who need to support more tables and views should increase the flag and ensure both vttablet and mysqld have enough memory to comfortably hold the larger schema.
 
 See [#19978](https://github.com/vitessio/vitess/issues/19978) for details.
+
+### <a id="minor-changes-backup"/>Backup/Restore</a>
+
+#### <a id="backup-chunked-builtin"/>Chunked backup/restore for the `builtinbackupengine`</a>
+
+The builtin backup engine now supports splitting large files into chunks for parallel backup and restore. This significantly improves restore throughput for keyspaces dominated by a small number of large InnoDB files, as individual chunks can be restored concurrently via parallel writes.
+
+Two new flags control chunking behavior:
+
+- `--builtinbackup-file-chunk-threshold` (default `0`, chunking disabled): files larger than this size in bytes are split into chunks during backup.
+- `--builtinbackup-file-chunk-size` (default `1073741824` / 1 GiB): the target size in bytes for each chunk.
+
+**Compatibility note:** Backups created with chunking enabled are **not restorable by older Vitess versions** that do not understand the `Chunks` field in the backup MANIFEST. Non-chunked backups (the default) remain fully compatible with older versions.
+
+See [#20167](https://github.com/vitessio/vitess/pull/20167) for details.

go/flags/endtoend/vtbackup.txt

@@ -61,6 +61,8 @@ Flags:
       --backup-storage-implementation string                        Which backup storage implementation to use for creating and restoring backups.
       --backup-storage-number-blocks int                            if backup-storage-compress is true, backup-storage-number-blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2)
       --bind-address string                                         Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system.
+      --builtinbackup-file-chunk-size int                           Size of each chunk (in bytes) when splitting large files for parallel backup/restore. (default 1073741824)
+      --builtinbackup-file-chunk-threshold int                      Files larger than this size (in bytes) are split into chunks for parallel backup/restore. 0 disables chunking.
       --builtinbackup-file-read-buffer-size uint                    read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.
       --builtinbackup-file-write-buffer-size uint                   write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152)
       --builtinbackup-incremental-restore-path string               the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.

go/flags/endtoend/vtcombo.txt

@@ -29,6 +29,8 @@ Flags:
       --buffer-min-time-between-failovers duration                       Minimum time between the end of a failover and the start of the next one (tracked per shard). Faster consecutive failovers will not trigger buffering. (default 1m0s)
       --buffer-size int                                                  Maximum number of buffered requests in flight (across all ongoing failovers). (default 1000)
       --buffer-window duration                                           Duration for how long a request should be buffered at most (should not be larger than --buffer-max-failover-duration). (default 10s)
+      --builtinbackup-file-chunk-size int                                Size of each chunk (in bytes) when splitting large files for parallel backup/restore. (default 1073741824)
+      --builtinbackup-file-chunk-threshold int                           Files larger than this size (in bytes) are split into chunks for parallel backup/restore. 0 disables chunking.
       --builtinbackup-file-read-buffer-size uint                         read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.
       --builtinbackup-file-write-buffer-size uint                        write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152)
       --builtinbackup-incremental-restore-path string                    the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.

go/flags/endtoend/vtctld.txt

@@ -34,6 +34,8 @@ Flags:
       --backup-storage-implementation string                             Which backup storage implementation to use for creating and restoring backups.
       --backup-storage-number-blocks int                                 if backup-storage-compress is true, backup-storage-number-blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2)
       --bind-address string                                              Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system.
+      --builtinbackup-file-chunk-size int                                Size of each chunk (in bytes) when splitting large files for parallel backup/restore. (default 1073741824)
+      --builtinbackup-file-chunk-threshold int                           Files larger than this size (in bytes) are split into chunks for parallel backup/restore. 0 disables chunking.
       --builtinbackup-file-read-buffer-size uint                         read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.
       --builtinbackup-file-write-buffer-size uint                        write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152)
       --builtinbackup-incremental-restore-path string                    the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.

go/flags/endtoend/vttablet.txt

@@ -62,6 +62,8 @@ Flags:
       --binlog-player-grpc-key string                                    the key to use to connect
       --binlog-player-grpc-server-name string                            the server name to use to validate server certificate
       --binlog-player-protocol string                                    the protocol to download binlogs from a vttablet (default "grpc")
+      --builtinbackup-file-chunk-size int                                Size of each chunk (in bytes) when splitting large files for parallel backup/restore. (default 1073741824)
+      --builtinbackup-file-chunk-threshold int                           Files larger than this size (in bytes) are split into chunks for parallel backup/restore. 0 disables chunking.
       --builtinbackup-file-read-buffer-size uint                         read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.
       --builtinbackup-file-write-buffer-size uint                        write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152)
       --builtinbackup-incremental-restore-path string                    the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.

go/flags/endtoend/vttestserver.txt

@@ -10,6 +10,8 @@ Flags:
       --backup-storage-block-size int                                    if backup-storage-compress is true, backup-storage-block-size sets the byte size for each block while compressing (default is 250000). (default 250000)
       --backup-storage-compress                                          if set, the backup files will be compressed. (default true)
       --backup-storage-number-blocks int                                 if backup-storage-compress is true, backup-storage-number-blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2)
+      --builtinbackup-file-chunk-size int                                Size of each chunk (in bytes) when splitting large files for parallel backup/restore. (default 1073741824)
+      --builtinbackup-file-chunk-threshold int                           Files larger than this size (in bytes) are split into chunks for parallel backup/restore. 0 disables chunking.
       --builtinbackup-file-read-buffer-size uint                         read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.
       --builtinbackup-file-write-buffer-size uint                        write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152)
       --builtinbackup-incremental-restore-path string                    the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.

go/test/endtoend/backup/vtctlbackup/backup_test.go

@@ -19,6 +19,8 @@ package vtctlbackup
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"vitess.io/vitess/go/vt/mysqlctl"
 )
 
@@ -67,6 +69,34 @@ func TestBuiltinBackupWithExternalZstdCompressionAndManifestedDecompressor(t *te
 	TestBackup(t, BuiltinBackup, "xbstream", 0, cDetails, []string{"TestReplicaBackup", "TestPrimaryBackup"})
 }
 
+// TestBuiltinBackupChunked uses a low chunk threshold (1MB) and chunk size (512KB) to force
+// InnoDB files (ibdata1, undo tablespaces) to be split into multiple chunks during backup.
+// This verifies that MySQL can start successfully and read back rows after a chunked restore.
+func TestBuiltinBackupChunked(t *testing.T) {
+	defer setDefaultCommonArgs()
+	commonTabletArg = append(
+		getDefaultCommonArgs(),
+		"--builtinbackup-file-chunk-threshold", "1048576",
+		"--builtinbackup-file-chunk-size", "524288",
+	)
+
+	code, err := LaunchCluster(BuiltinBackup, "xbstream", 0, nil)
+	require.Nilf(t, err, "setup failed with status code %d", code)
+	defer TearDownCluster()
+
+	t.Run("TestChunkedBackup", chunkedBackup)
+}
+
+// TestBuiltinBackupNonChunked verifies that with the default threshold of 0, no chunking
+// occurs and the MANIFEST remains compatible with older Vitess versions.
+func TestBuiltinBackupNonChunked(t *testing.T) {
+	code, err := LaunchCluster(BuiltinBackup, "xbstream", 0, nil)
+	require.Nilf(t, err, "setup failed with status code %d", code)
+	defer TearDownCluster()
+
+	t.Run("TestNonChunkedBackup", nonChunkedBackup)
+}
+
 func setDefaultCompressionFlag() {
 	mysqlctl.CompressionEngineName = "pgzip"
 	mysqlctl.ExternalCompressorCmd = ""

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -977,6 +977,68 @@ func vtctlBackup(t *testing.T, tabletType string) {
 	require.NoError(t, err)
 }
 
+func chunkedBackup(t *testing.T) {
+	verifyBackupChunking(t, true)
+}
+
+func nonChunkedBackup(t *testing.T) {
+	verifyBackupChunking(t, false)
+}
+
+func verifyBackupChunking(t *testing.T, expectChunked bool) {
+	verifyInitialReplication(t)
+
+	cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1)
+
+	restoreWaitForBackup(t, "replica", nil, true)
+
+	err := localCluster.VtctldClientProcess.ExecuteCommand("Backup", replica1.Alias)
+	require.NoError(t, err)
+
+	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
+	backupLocation := path.Join(localCluster.CurrentVTDATAROOT, "backups", keyspaceName, shardName, backups[0])
+
+	manifestData, err := os.ReadFile(path.Join(backupLocation, "MANIFEST"))
+	require.NoError(t, err)
+
+	var manifest struct {
+		FileEntries []struct {
+			Name   string
+			Chunks []struct {
+				StorageName string
+			}
+		}
+	}
+	require.NoError(t, json.Unmarshal(manifestData, &manifest))
+
+	totalChunks := 0
+	for _, fe := range manifest.FileEntries {
+		if len(fe.Chunks) > 0 {
+			t.Logf("File %s: %d chunks", fe.Name, len(fe.Chunks))
+			totalChunks += len(fe.Chunks)
+		}
+	}
+	t.Logf("Total chunks in manifest: %d", totalChunks)
+
+	if expectChunked {
+		assert.Greater(t, totalChunks, 0, "expected at least one file to be chunked")
+	} else {
+		assert.Equal(t, 0, totalChunks, "expected no chunking with threshold=0")
+	}
+
+	err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second)
+	require.NoError(t, err)
+	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 1)
+
+	verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups)
+	err = replica2.VttabletProcess.TearDown()
+	require.NoError(t, err)
+	err = localCluster.VtctldClientProcess.ExecuteCommand("DeleteTablets", replica2.Alias)
+	require.NoError(t, err)
+	_, err = primary.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true)
+	require.NoError(t, err)
+}
+
 func InitTestTable(t *testing.T) {
 	_, err := primary.VttabletProcess.QueryTablet("DROP TABLE IF EXISTS vt_insert_test", keyspaceName, true)
 	require.NoError(t, err)