Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go/flags/endtoend/vtbackup.txt
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ Flags:
--external-compressor string command with arguments to use when compressing a backup.
--external-compressor-extension string extension to use when using an external compressor.
--external-decompressor string command with arguments to use when decompressing a backup.
--external-decompressor-use-manifest allows the decompressor command stored in the backup manifest to be used at restore time. Enabling this is a security risk: an attacker with write access to the backup storage could modify the manifest to execute arbitrary commands on the tablet as the Vitess user. NOT RECOMMENDED.
--file-backup-storage-root string Root directory for the file backup storage.
--gcs-backup-storage-bucket string Google Cloud Storage bucket to use for backups.
--gcs-backup-storage-root string Root prefix for all backup-related object names.
Expand Down
1 change: 1 addition & 0 deletions go/flags/endtoend/vtcombo.txt
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ Flags:
--external-compressor string command with arguments to use when compressing a backup.
--external-compressor-extension string extension to use when using an external compressor.
--external-decompressor string command with arguments to use when decompressing a backup.
--external-decompressor-use-manifest allows the decompressor command stored in the backup manifest to be used at restore time. Enabling this is a security risk: an attacker with write access to the backup storage could modify the manifest to execute arbitrary commands on the tablet as the Vitess user. NOT RECOMMENDED.
--external-topo-server Should vtcombo use an external topology server instead of starting its own in-memory topology server. If true, vtcombo will use the flags defined in topo/server.go to open topo server
--foreign-key-mode string This is to provide how to handle foreign key constraint in create/alter table. Valid values are: allow, disallow (default "allow")
--gate-query-cache-memory int gate server query cache size in bytes, maximum amount of memory to be cached. vtgate analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache. (default 33554432)
Expand Down
1 change: 1 addition & 0 deletions go/flags/endtoend/vttablet.txt
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ Flags:
--external-compressor string command with arguments to use when compressing a backup.
--external-compressor-extension string extension to use when using an external compressor.
--external-decompressor string command with arguments to use when decompressing a backup.
--external-decompressor-use-manifest allows the decompressor command stored in the backup manifest to be used at restore time. Enabling this is a security risk: an attacker with write access to the backup storage could modify the manifest to execute arbitrary commands on the tablet as the Vitess user. NOT RECOMMENDED.
--file-backup-storage-root string Root directory for the file backup storage.
--filecustomrules string file based custom rule path
--filecustomrules-watch set up a watch on the target file and reload query rules when it changes
Expand Down
1 change: 1 addition & 0 deletions go/flags/endtoend/vttestserver.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ Flags:
--external-compressor string command with arguments to use when compressing a backup.
--external-compressor-extension string extension to use when using an external compressor.
--external-decompressor string command with arguments to use when decompressing a backup.
--external-decompressor-use-manifest allows the decompressor command stored in the backup manifest to be used at restore time. Enabling this is a security risk: an attacker with write access to the backup storage could modify the manifest to execute arbitrary commands on the tablet as the Vitess user. NOT RECOMMENDED.
--external-topo-global-root string the path of the global topology data in the global topology server for vtcombo process
--external-topo-global-server-address string the address of the global topology server for vtcombo process
--external-topo-implementation string the topology implementation to use for vtcombo process
Expand Down
1 change: 1 addition & 0 deletions go/test/endtoend/backup/vtctlbackup/backup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ func TestBuiltinBackupWithExternalZstdCompressionAndManifestedDecompressor(t *te
CompressorEngineName: "external",
ExternalCompressorCmd: "zstd",
ExternalCompressorExt: ".zst",
ExternalDecompressorUseManifest: true,
ManifestExternalDecompressorCmd: "zstd -d",
}

Expand Down
4 changes: 4 additions & 0 deletions go/test/endtoend/backup/vtctlbackup/backup_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ type CompressionDetails struct {
ExternalCompressorCmd string
ExternalCompressorExt string
ExternalDecompressorCmd string
ExternalDecompressorUseManifest bool
ManifestExternalDecompressorCmd string
}

Expand Down Expand Up @@ -307,6 +308,9 @@ func getCompressorArgs(cDetails *CompressionDetails) []string {
if cDetails.ExternalDecompressorCmd != "" {
args = append(args, "--external-decompressor="+cDetails.ExternalDecompressorCmd)
}
if cDetails.ExternalDecompressorUseManifest {
args = append(args, "--external-decompressor-use-manifest")
}
if cDetails.ManifestExternalDecompressorCmd != "" {
args = append(args, "--manifest-external-decompressor="+cDetails.ManifestExternalDecompressorCmd)
}
Expand Down
1 change: 1 addition & 0 deletions go/test/endtoend/backup/xtrabackup/xtrabackup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ func TestXtrabackupWithExternalZstdCompressionAndManifestedDecompressor(t *testi
CompressorEngineName: "external",
ExternalCompressorCmd: "zstd",
ExternalCompressorExt: ".zst",
ExternalDecompressorUseManifest: true,
ManifestExternalDecompressorCmd: "zstd -d",
}

Expand Down
5 changes: 1 addition & 4 deletions go/vt/mysqlctl/builtinbackupengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -1329,10 +1329,7 @@ func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, params RestorePa
// for backward compatibility
deCompressionEngine = PgzipCompressor
}
externalDecompressorCmd := ExternalDecompressorCmd
if externalDecompressorCmd == "" && bm.ExternalDecompressor != "" {
externalDecompressorCmd = bm.ExternalDecompressor
}
externalDecompressorCmd := resolveExternalDecompressor(bm.ExternalDecompressor)
if externalDecompressorCmd != "" {
if deCompressionEngine == ExternalCompressor {
deCompressionEngine = externalDecompressorCmd
Expand Down
18 changes: 17 additions & 1 deletion go/vt/mysqlctl/compression.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,10 @@ var (
ExternalCompressorCmd string
ExternalCompressorExt string
ExternalDecompressorCmd string
ExternalDecompressorUseManifest bool
ManifestExternalDecompressorCmd string

errUnsupportedDeCompressionEngine = errors.New("unsupported engine in MANIFEST. You need to provide --external-decompressor if using 'external' compression engine")
errUnsupportedDeCompressionEngine = errors.New("unsupported engine in MANIFEST. You need to provide --external-decompressor if using 'external' compression engine. Alternatively, set --external-decompressor-use-manifest to use the decompressor command from the backup manifest, but this is NOT RECOMMENDED as it is a security risk")
errUnsupportedCompressionEngine = errors.New("unsupported engine value for --compression-engine-name. supported values are 'external', 'pgzip', 'pargzip', 'zstd', 'lz4'")

// this is used by getEngineFromExtension() to figure out which engine to use in case the user didn't specify
Expand All @@ -78,6 +79,7 @@ func registerBackupCompressionFlags(fs *pflag.FlagSet) {
fs.StringVar(&ExternalCompressorCmd, "external-compressor", ExternalCompressorCmd, "command with arguments to use when compressing a backup.")
fs.StringVar(&ExternalCompressorExt, "external-compressor-extension", ExternalCompressorExt, "extension to use when using an external compressor.")
fs.StringVar(&ExternalDecompressorCmd, "external-decompressor", ExternalDecompressorCmd, "command with arguments to use when decompressing a backup.")
fs.BoolVar(&ExternalDecompressorUseManifest, "external-decompressor-use-manifest", ExternalDecompressorUseManifest, "allows the decompressor command stored in the backup manifest to be used at restore time. Enabling this is a security risk: an attacker with write access to the backup storage could modify the manifest to execute arbitrary commands on the tablet as the Vitess user. NOT RECOMMENDED.")
fs.StringVar(&ManifestExternalDecompressorCmd, "manifest-external-decompressor", ManifestExternalDecompressorCmd, "command with arguments to store in the backup manifest when compressing a backup with an external compression engine.")
}

Expand All @@ -90,6 +92,20 @@ func getExtensionFromEngine(engine string) (string, error) {
return "", fmt.Errorf("%w %q", errUnsupportedCompressionEngine, engine)
}

// resolveExternalDecompressor returns the external decompressor command to use
// at restore time. The CLI flag (--external-decompressor) takes precedence. The
// backup manifest value is only used when --external-decompressor-use-manifest
// is explicitly set to true.
func resolveExternalDecompressor(manifestDecompressor string) string {
if ExternalDecompressorCmd != "" {
return ExternalDecompressorCmd
}
if ExternalDecompressorUseManifest && manifestDecompressor != "" {
return manifestDecompressor
}
return ""
}

// Validates if the external decompressor exists and return its path.
func validateExternalCmd(cmd string) (string, error) {
if cmd == "" {
Expand Down
93 changes: 93 additions & 0 deletions go/vt/mysqlctl/compression_external_decompressor_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
Copyright 2026 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysqlctl

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestResolveExternalDecompressor(t *testing.T) {
tests := []struct {
name string
cliDecompressorCmd string
useManifest bool
manifestDecompressor string
expected string
}{
{
name: "CLI flag takes precedence over manifest",
cliDecompressorCmd: "zstd -d",
useManifest: true,
manifestDecompressor: "gzip -d",
expected: "zstd -d",
},
{
name: "CLI flag takes precedence even when use-manifest is false",
cliDecompressorCmd: "zstd -d",
useManifest: false,
manifestDecompressor: "gzip -d",
expected: "zstd -d",
},
{
name: "manifest used when use-manifest is true and no CLI flag",
cliDecompressorCmd: "",
useManifest: true,
manifestDecompressor: "gzip -d",
expected: "gzip -d",
},
{
name: "manifest ignored when use-manifest is false",
cliDecompressorCmd: "",
useManifest: false,
manifestDecompressor: "gzip -d",
expected: "",
},
{
name: "empty when nothing is set",
cliDecompressorCmd: "",
useManifest: false,
manifestDecompressor: "",
expected: "",
},
{
name: "empty when use-manifest is true but manifest is empty",
cliDecompressorCmd: "",
useManifest: true,
manifestDecompressor: "",
expected: "",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
origCmd := ExternalDecompressorCmd
origAllow := ExternalDecompressorUseManifest
t.Cleanup(func() {
ExternalDecompressorCmd = origCmd
ExternalDecompressorUseManifest = origAllow
})

ExternalDecompressorCmd = tt.cliDecompressorCmd
ExternalDecompressorUseManifest = tt.useManifest

result := resolveExternalDecompressor(tt.manifestDecompressor)
assert.Equal(t, tt.expected, result)
})
}
}
5 changes: 1 addition & 4 deletions go/vt/mysqlctl/xtrabackupengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -649,10 +649,7 @@ func (be *XtrabackupEngine) extractFiles(ctx context.Context, logger logutil.Log
// then we assign the default value of compressionEngine.
deCompressionEngine = PgzipCompressor
}
externalDecompressorCmd := ExternalDecompressorCmd
if externalDecompressorCmd == "" && bm.ExternalDecompressor != "" {
externalDecompressorCmd = bm.ExternalDecompressor
}
externalDecompressorCmd := resolveExternalDecompressor(bm.ExternalDecompressor)
if externalDecompressorCmd != "" {
if deCompressionEngine == ExternalCompressor {
deCompressionEngine = externalDecompressorCmd
Expand Down
Loading