diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 8483349188d..b9ce3cbe3a9 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -278,6 +278,7 @@ Flags: --proxy-tablets Setting this true will make vtctld proxy the tablet status instead of redirecting to them --publish-retry-interval duration how long vttablet waits to retry publishing the tablet record (default 30s) --purge-logs-interval duration how often try to remove old logs (default 1h0m0s) + --query-log-ignore-patterns string Comma-separated list of SQL query shapes to suppress from the vtgate query log. Patterns are matched against the normalized query shape (literals replaced with bind placeholders, e.g. 'select :vtg1'); unparseable patterns fall back to case-insensitive trimmed raw-string match. Prefix the value with '@' to read patterns from a file (one per line; '#' starts a comment). Empty by default. --query-log-stream-handler string URL handler for streaming queries log (default "/debug/querylog") --query-throttler-config-refresh-interval duration How frequently to refresh configuration for the query throttler (default 1m0s) --query-timeout int Sets the default query timeout (in ms). Can be overridden by session variable (query_timeout) or comment directive (QUERY_TIMEOUT_MS) diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 997ec68eb15..8bc5c08d887 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -176,6 +176,7 @@ Flags: --prevent-cross-keyspace-reads when set to true, the planner will fail instead of producing a plan that includes cross-keyspace joins or UNIONs --proxy-protocol Enable HAProxy PROXY protocol on MySQL listener socket --purge-logs-interval duration how often try to remove old logs (default 1h0m0s) + --query-log-ignore-patterns string Comma-separated list of SQL query shapes to suppress from the vtgate query log. Patterns are matched against the normalized query shape (literals replaced with bind placeholders, e.g. 'select :vtg1'); unparseable patterns fall back to case-insensitive trimmed raw-string match. Prefix the value with '@' to read patterns from a file (one per line; '#' starts a comment). Empty by default. --query-timeout int Sets the default query timeout (in ms). Can be overridden by session variable (query_timeout) or comment directive (QUERY_TIMEOUT_MS) --querylog-buffer-size int Maximum number of buffered query logs before throttling log output (default 10) --querylog-emit-on-any-condition-met Emit to query log when any of the conditions (row-threshold, time-threshold, filter-tag) is met (default false) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index cd49508feb8..32ef1f7ecc8 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -69,6 +69,7 @@ import ( "vitess.io/vitess/go/vt/vtgate/logstats" "vitess.io/vitess/go/vt/vtgate/planbuilder" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + "vitess.io/vitess/go/vt/vtgate/querylogignore" "vitess.io/vitess/go/vt/vtgate/vindexes" "vitess.io/vitess/go/vt/vtgate/vschemaacl" "vitess.io/vitess/go/vt/vtgate/vtgateservice" @@ -287,7 +288,7 @@ func (e *Executor) Execute( } logStats.SaveEndTime() - e.queryLogger.Send(logStats) + e.sendQueryLog(logStats) err = errorTransform.TransformError(err) err = vterrors.TruncateError(err, truncateErrorLen) @@ -433,7 +434,7 @@ func (e *Executor) StreamExecute( } logStats.SaveEndTime() - e.queryLogger.Send(logStats) + e.sendQueryLog(logStats) err = errorTransform.TransformError(err) err = vterrors.TruncateError(err, truncateErrorLen) @@ -441,6 +442,13 @@ func (e *Executor) StreamExecute( return err } +func (e *Executor) sendQueryLog(logStats *logstats.LogStats) { + if querylogignore.IgnorePatterns.Get().ShouldIgnore(logStats.SQL, e.env.Parser()) { + return + } + e.queryLogger.Send(logStats) +} + func canReturnRows(stmtType sqlparser.StatementType) bool { switch stmtType { case sqlparser.StmtSelect, sqlparser.StmtShow, sqlparser.StmtExplain, sqlparser.StmtCallProc: @@ -1508,7 +1516,7 @@ func (e *Executor) Prepare(ctx context.Context, method string, safeSession *econ // it was a no-op record (i.e. didn't issue any queries) if logStats.StmtType != "ROLLBACK" || logStats.ShardQueries != 0 { logStats.SaveEndTime() - e.queryLogger.Send(logStats) + e.sendQueryLog(logStats) } err = errorTransform.TransformError(err) diff --git a/go/vt/vtgate/executor_querylog_ignore_test.go b/go/vt/vtgate/executor_querylog_ignore_test.go new file mode 100644 index 00000000000..f3428c0ee8b --- /dev/null +++ b/go/vt/vtgate/executor_querylog_ignore_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtgate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" + "vitess.io/vitess/go/vt/vtgate/querylogignore" +) + +// TestExecutorQueryLogIgnore verifies that the executor suppresses query-log +// emission for SQL whose normalized shape matches an operator-configured +// pattern, while still emitting non-matching queries. +func TestExecutorQueryLogIgnore(t *testing.T) { + executor, _, _, _, ctx := createExecutorEnv(t) + + // Configure the ignore-list and reset it after the test. + parser := executor.env.Parser() + original := querylogignore.IgnorePatterns.Get() + querylogignore.IgnorePatterns.Set(querylogignore.NewIgnoreSet("select id from main1", parser)) + t.Cleanup(func() { + querylogignore.IgnorePatterns.Set(original) + }) + + logChan := executor.queryLogger.Subscribe("TestExecutorQueryLogIgnore") + defer executor.queryLogger.Unsubscribe(logChan) + + session := &vtgatepb.Session{TargetString: "@primary", Autocommit: true} + + // A query matching the configured shape (different literal still matches + // the canonical form) must not produce a log entry. + _, err := executorExec(ctx, executor, session, "select id from main1", nil) + require.NoError(t, err) + assert.Nil(t, getQueryLog(logChan), "matching query must be suppressed") + + // A non-matching query must still be logged. + _, err = executorExec(ctx, executor, session, "select name from user", nil) + require.NoError(t, err) + logStats := getQueryLog(logChan) + require.NotNil(t, logStats, "non-matching query must still be logged") + assert.Contains(t, logStats.SQL, "name") +} diff --git a/go/vt/vtgate/querylogignore/querylogignore.go b/go/vt/vtgate/querylogignore/querylogignore.go new file mode 100644 index 00000000000..d32a8bf1307 --- /dev/null +++ b/go/vt/vtgate/querylogignore/querylogignore.go @@ -0,0 +1,232 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package querylogignore implements an opt-in allow/deny mechanism for +// suppressing specific SQL query shapes from the vtgate query log. +// +// Operators configure a list of patterns via --query-log-ignore-patterns. +// Each pattern is matched against the normalized shape of an incoming query +// (literals replaced with bind placeholders, e.g. "select :vtg1"), with a +// case-insensitive trimmed raw-string fallback for queries that don't parse. +package querylogignore + +import ( + "log/slog" + "os" + "regexp" + "strings" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/sqlparser" +) + +// filePrefix marks the flag value as a path to a file containing one pattern +// per line (blank lines and lines starting with "#" are skipped). +const filePrefix = "@" + +// typeCommentRE strips the " /* INT64 */"-style type annotations that +// sqlparser.Normalize attaches to each replaced literal. Without this, +// patterns supplied in bind-variable form would not match queries whose +// literals were normalized away. +var typeCommentRE = regexp.MustCompile(` /\* [A-Z0-9_]+ \*/`) + +// bindVarRE collapses any bind-variable reference (`:vtg1`, `:redacted1`, +// `:x`, ...) to a single placeholder so that patterns written in any of +// the conventional bind-variable styles compare equal. +var bindVarRE = regexp.MustCompile(`:[a-zA-Z_][a-zA-Z0-9_]*`) + +// canonicalize reduces a redacted SQL string to a form that ignores both +// type annotations and the specific bind-variable names chosen by the +// normalizer, so equivalent shapes match regardless of how the operator +// wrote the pattern. +func canonicalize(redacted string) string { + redacted = typeCommentRE.ReplaceAllString(redacted, "") + redacted = bindVarRE.ReplaceAllString(redacted, ":_") + return redacted +} + +// IgnoreSet is the parsed form of --query-log-ignore-patterns. It contains +// both normalized and raw-fallback forms of every configured pattern so a +// single map lookup per tier answers ShouldIgnore. +type IgnoreSet struct { + // set holds the normalized form of each parseable pattern, plus the + // trimmed-lowercased raw form of patterns that fail to parse. The empty + // case is the common one and is detected via len(set) == 0 on the hot + // path. + set map[string]struct{} + + // source preserves the original flag value so the viper GetFunc can + // short-circuit re-parsing when the underlying string hasn't changed. + source string +} + +// NewIgnoreSet builds an IgnoreSet from the raw flag value. The value is +// either a comma-separated list of patterns or, if it begins with "@", a +// path to a file containing one pattern per line. Patterns that fail to +// parse are stored as their trimmed lower-cased raw form, so unparseable +// queries like "select $$" can still be matched. +func NewIgnoreSet(rawValue string, parser *sqlparser.Parser) *IgnoreSet { + s := &IgnoreSet{source: rawValue} + + patterns := loadPatterns(rawValue) + if len(patterns) == 0 { + return s + } + + s.set = make(map[string]struct{}, len(patterns)) + for _, p := range patterns { + p = strings.TrimSpace(p) + if p == "" { + continue + } + if parser != nil { + if normalized, err := parser.RedactSQLQuery(p); err == nil { + s.set[canonicalize(normalized)] = struct{}{} + continue + } + } + s.set[strings.ToLower(p)] = struct{}{} + } + return s +} + +// loadPatterns returns the raw pattern strings from the flag value, reading +// from disk when the value begins with "@". A read error is logged and +// returns an empty slice so vtgate still starts. +func loadPatterns(rawValue string) []string { + rawValue = strings.TrimSpace(rawValue) + if rawValue == "" { + return nil + } + if strings.HasPrefix(rawValue, filePrefix) { + path := rawValue[len(filePrefix):] + data, err := os.ReadFile(path) + if err != nil { + log.Warn( + "query-log-ignore-patterns: failed to read file; ignore-list will be empty", + slog.String("path", path), + slog.Any("error", err), + ) + return nil + } + var out []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out = append(out, line) + } + return out + } + return strings.Split(rawValue, ",") +} + +// ShouldIgnore reports whether the given SQL string matches any configured +// pattern. The empty-set check is a single integer comparison so operators +// who do not configure the flag pay essentially no cost. +func (s *IgnoreSet) ShouldIgnore(sql string, parser *sqlparser.Parser) bool { + if s == nil || len(s.set) == 0 { + return false + } + if parser != nil { + if normalized, err := parser.RedactSQLQuery(sql); err == nil { + if _, ok := s.set[canonicalize(normalized)]; ok { + return true + } + } + } + raw := strings.ToLower(strings.TrimSpace(sql)) + _, ok := s.set[raw] + return ok +} + +// Source returns the original flag string used to build this IgnoreSet. +func (s *IgnoreSet) Source() string { + if s == nil { + return "" + } + return s.source +} + +// String implements fmt.Stringer so that viper's GetString returns the +// underlying source. The dynamic GetFunc relies on this when comparing the +// freshly-read config value against the cached IgnoreSet's source. +func (s *IgnoreSet) String() string { + return s.Source() +} + +// flagParser is used to normalize patterns at flag-parse time. The executor +// passes its own parser at lookup time; this one only handles the small set +// of configured patterns, so the default MySQL server version is sufficient. +var flagParser = mustNewParser() + +func mustNewParser() *sqlparser.Parser { + p, err := sqlparser.New(sqlparser.Options{}) + if err != nil { + log.Error( + "query-log-ignore-patterns: failed to construct flag parser; parseable shape matching disabled", + slog.Any("error", err), + ) + return nil + } + return p +} + +// IgnorePatterns is the Viper-managed flag value backing +// --query-log-ignore-patterns. Empty by default. Reloadable via Viper. +var IgnorePatterns = viperutil.Configure( + "query_log_ignore_patterns", + viperutil.Options[*IgnoreSet]{ + FlagName: "query-log-ignore-patterns", + Default: &IgnoreSet{}, + Dynamic: true, + GetFunc: func(v *viper.Viper) func(key string) *IgnoreSet { + return func(key string) *IgnoreSet { + newVal := v.GetString(key) + if cur, ok := v.Get(key).(*IgnoreSet); ok && cur.source == newVal { + return cur + } + return NewIgnoreSet(newVal, flagParser) + } + }, + }, +) + +// RegisterFlags installs --query-log-ignore-patterns on the given FlagSet. +func RegisterFlags(fs *pflag.FlagSet) { + fs.String( + "query-log-ignore-patterns", + "", + "Comma-separated list of SQL query shapes to suppress from the vtgate query log. "+ + "Patterns are matched against the normalized query shape (literals replaced with bind placeholders, "+ + "e.g. 'select :vtg1'); unparseable patterns fall back to case-insensitive trimmed raw-string match. "+ + "Prefix the value with '@' to read patterns from a file (one per line; '#' starts a comment). "+ + "Empty by default.", + ) + viperutil.BindFlags(fs, IgnorePatterns) +} + +func init() { + for _, cmd := range []string{"vtgate", "vtcombo"} { + servenv.OnParseFor(cmd, RegisterFlags) + } +} diff --git a/go/vt/vtgate/querylogignore/querylogignore_test.go b/go/vt/vtgate/querylogignore/querylogignore_test.go new file mode 100644 index 00000000000..440deaa3302 --- /dev/null +++ b/go/vt/vtgate/querylogignore/querylogignore_test.go @@ -0,0 +1,171 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package querylogignore + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/vt/sqlparser" +) + +func newParser(t *testing.T) *sqlparser.Parser { + t.Helper() + p, err := sqlparser.New(sqlparser.Options{}) + require.NoError(t, err) + return p +} + +func TestShouldIgnore_EmptyFastPath(t *testing.T) { + parser := newParser(t) + s := NewIgnoreSet("", parser) + + assert.False(t, s.ShouldIgnore("select 1", parser)) + assert.False(t, s.ShouldIgnore("select $$", parser)) + assert.False(t, s.ShouldIgnore("", parser)) + + // nil receiver must also be safe and return false. + var nilSet *IgnoreSet + assert.False(t, nilSet.ShouldIgnore("select 1", parser)) +} + +func TestShouldIgnore_NormalizedShapeMatchesLiterals(t *testing.T) { + parser := newParser(t) + s := NewIgnoreSet("select :vtg1 from dual", parser) + + assert.True(t, s.ShouldIgnore("select 1 from dual", parser)) + assert.True(t, s.ShouldIgnore("select 9999 from dual", parser)) + assert.True(t, s.ShouldIgnore("SELECT 7 FROM dual", parser)) +} + +func TestShouldIgnore_SelectDollarVariants(t *testing.T) { + // The motivating example from the issue. Current sqlparser parses + // "select $$" successfully (it normalizes to "select $$ from dual"), + // so all whitespace/case variants land in the same canonical bucket. + parser := newParser(t) + s := NewIgnoreSet("select $$", parser) + + assert.True(t, s.ShouldIgnore("select $$", parser)) + assert.True(t, s.ShouldIgnore(" select $$ ", parser)) + assert.True(t, s.ShouldIgnore("SELECT $$", parser)) + assert.True(t, s.ShouldIgnore("select $$ from dual", parser)) +} + +func TestShouldIgnore_UnparseableRawFallback(t *testing.T) { + // A truly unparseable input (no SELECT/INSERT prefix) goes through the + // raw-string fallback path. Verify the configured pattern matches + // itself, plus whitespace-trimmed and case-insensitive variants. + parser := newParser(t) + s := NewIgnoreSet("PING", parser) + + assert.True(t, s.ShouldIgnore("PING", parser)) + assert.True(t, s.ShouldIgnore(" ping ", parser)) + assert.True(t, s.ShouldIgnore("Ping", parser)) +} + +func TestShouldIgnore_DistinctShapesDiffer(t *testing.T) { + parser := newParser(t) + s := NewIgnoreSet("select $$", parser) + + // A genuinely different shape must not match. + assert.False(t, s.ShouldIgnore("select $$ from mytable", parser)) + assert.False(t, s.ShouldIgnore("select $$, $$ from dual", parser)) +} + +func TestShouldIgnore_NonMatchingQueriesLogNormally(t *testing.T) { + parser := newParser(t) + s := NewIgnoreSet("select :vtg1", parser) + + assert.False(t, s.ShouldIgnore("insert into t values (1)", parser)) + assert.False(t, s.ShouldIgnore("select id from t where x = 1", parser)) +} + +func TestNewIgnoreSet_MultiplePatterns(t *testing.T) { + parser := newParser(t) + s := NewIgnoreSet("select :vtg1,select $$,select :vtg1 from dual", parser) + + assert.True(t, s.ShouldIgnore("select 1", parser)) + assert.True(t, s.ShouldIgnore("select $$", parser)) + assert.True(t, s.ShouldIgnore("select 42 from dual", parser)) + assert.False(t, s.ShouldIgnore("update t set a = 1", parser)) +} + +func TestNewIgnoreSet_FromFile(t *testing.T) { + parser := newParser(t) + + path := filepath.Join(t.TempDir(), "ignore.txt") + contents := `# health-check pings +select $$ + +# raw selects of any literal +select :vtg1 +` + require.NoError(t, os.WriteFile(path, []byte(contents), 0o644)) + + s := NewIgnoreSet("@"+path, parser) + + assert.True(t, s.ShouldIgnore("select $$", parser)) + assert.True(t, s.ShouldIgnore("select 99", parser)) + assert.False(t, s.ShouldIgnore("delete from t", parser)) +} + +func TestNewIgnoreSet_MissingFileWarnsAndContinues(t *testing.T) { + parser := newParser(t) + + missing := filepath.Join(t.TempDir(), "does-not-exist.txt") + s := NewIgnoreSet("@"+missing, parser) + + // Constructor must not panic; ShouldIgnore returns false for everything. + assert.False(t, s.ShouldIgnore("select $$", parser)) + assert.False(t, s.ShouldIgnore("select 1", parser)) +} + +func TestNewIgnoreSet_TrimsAndSkipsEmptyEntries(t *testing.T) { + parser := newParser(t) + s := NewIgnoreSet(" select :vtg1 ,, , select $$ ", parser) + + assert.True(t, s.ShouldIgnore("select 1", parser)) + assert.True(t, s.ShouldIgnore("select $$", parser)) +} + +func TestIgnoreSet_SourceRoundTrip(t *testing.T) { + parser := newParser(t) + raw := "select :vtg1,select $$" + s := NewIgnoreSet(raw, parser) + assert.Equal(t, raw, s.Source()) + + var nilSet *IgnoreSet + assert.Equal(t, "", nilSet.Source()) +} + +func BenchmarkShouldIgnore_EmptyFastPath(b *testing.B) { + parser, err := sqlparser.New(sqlparser.Options{}) + require.NoError(b, err) + s := NewIgnoreSet("", parser) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if s.ShouldIgnore("select id from t where x = 42", parser) { + b.Fatal("expected false on empty set") + } + } +}