Skip to content

evalengine: make JSON_EXTRACT work with non-static arguments #19035

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arthurschreiber merged 8 commits into from
Dec 17, 2025
Merged

evalengine: make JSON_EXTRACT work with non-static arguments #19035

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d44393b
Make `JSON_EXTRACT` work with non-static arguments.
arthurschreiber Dec 16, 2025
d16406f
Split up test cases into one per function.
arthurschreiber Dec 16, 2025
4153731
Make compiled `JSON_EXTRACT` calls handle first argument being `NULL`
arthurschreiber Dec 16, 2025
52fbaf5
Add current JSON path errors to list of known differences.
arthurschreiber Dec 16, 2025
d32b8c7
Small tweak.
arthurschreiber Dec 16, 2025
5c4dabe
Fix handling of static values from functions.
arthurschreiber Dec 16, 2025
89dcdbc
Fix typo.
arthurschreiber Dec 16, 2025
3f12ed4
Update comment.
arthurschreiber Dec 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 65 additions & 17 deletions go/vt/vtgate/evalengine/compiler_asm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ import (
"vitess.io/vitess/go/mysql/hex"
"vitess.io/vitess/go/mysql/icuregex"
"vitess.io/vitess/go/mysql/json"
"vitess.io/vitess/go/slice"
"vitess.io/vitess/go/sqltypes"
querypb "vitess.io/vitess/go/vt/proto/query"
"vitess.io/vitess/go/vt/proto/vtrpc"
Expand Down Expand Up @@ -2215,40 +2214,89 @@ func (asm *assembler) Fn_JSON_CONTAINS_PATH(match jsonMatch, paths []*json.Path)
}
}

func (asm *assembler) Fn_JSON_EXTRACT0(jp []*json.Path) {
multi := len(jp) > 1 || slice.Any(jp, func(path *json.Path) bool { return path.ContainsWildcards() })
type staticPath struct {
p *json.Path
err error
}

if multi {
asm.emit(func(env *ExpressionEnv) int {
func (asm *assembler) Fn_JSON_EXTRACT(args int, staticPaths []staticPath) {
asm.adjustStack(-args)
asm.emit(func(env *ExpressionEnv) int {
paths := make([]*json.Path, 0, args)

multi := args > 1

doct := env.vm.stack[env.vm.sp-(args+1)].(*evalJSON)

for i := args; i > 0; i-- {
arg := env.vm.stack[env.vm.sp-i]

if arg == nil {
env.vm.sp -= args
env.vm.stack[env.vm.sp-1] = nil
return 1
}

staticPath := staticPaths[args-i]
if staticPath.err != nil {
env.vm.err = staticPath.err
return 1
}

var path *json.Path
if staticPath.p != nil {
path = staticPath.p
} else {
pathBytes, err := evalToVarchar(arg, collations.CollationUtf8mb4ID, true)
if err != nil {
env.vm.err = err
return 1
}

path, err = intoJSONPath(pathBytes)
if err != nil {
env.vm.err = err
return 1
}
}

if !multi {
multi = path.ContainsWildcards()
}

paths = append(paths, path)
}

env.vm.sp -= args

if multi {
matches := make([]*json.Value, 0, 4)
arg := env.vm.stack[env.vm.sp-1].(*evalJSON)
for _, jp := range jp {
jp.Match(arg, true, func(value *json.Value) {
for _, jp := range paths {
jp.Match(doct, true, func(value *json.Value) {
matches = append(matches, value)
})
}

if len(matches) == 0 {
env.vm.stack[env.vm.sp-1] = nil
} else {
env.vm.stack[env.vm.sp-1] = json.NewArray(matches)
}
return 1
}, "FN JSON_EXTRACT, SP-1, [static]")
} else {
asm.emit(func(env *ExpressionEnv) int {
} else {
var match *json.Value
arg := env.vm.stack[env.vm.sp-1].(*evalJSON)
jp[0].Match(arg, true, func(value *json.Value) {
paths[0].Match(doct, true, func(value *json.Value) {
match = value
})

if match == nil {
env.vm.stack[env.vm.sp-1] = nil
} else {
env.vm.stack[env.vm.sp-1] = match
}
return 1
}, "FN JSON_EXTRACT, SP-1, [static]")
}
}

return 1
}, "FN JSON_EXTRACT, SP-1, ..., SP-N")
}

func (asm *assembler) Fn_JSON_KEYS(jp *json.Path) {
Expand Down
56 changes: 43 additions & 13 deletions go/vt/vtgate/evalengine/fn_json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (call *builtinJSONExtract) eval(env *ExpressionEnv) (eval, error) {
}

func builtin_JSON_EXTRACT(doc *json.Value, paths []eval) (eval, error) {
matches := make([]*json.Value, 0, 4)
matches := make([]*json.Value, 0, len(paths))
multi := len(paths) > 1

for _, p := range paths {
Expand Down Expand Up @@ -125,27 +125,57 @@ func (call *builtinJSONExtract) compile(c *compiler) (ctype, error) {
return ctype{}, err
}

if slice.All(call.Arguments[1:], func(expr IR) bool { return expr.constant() }) {
paths := make([]*json.Path, 0, len(call.Arguments[1:]))
// Handle `NULL` arguments
nullable := doct.nullable()
skip := c.compileNullCheck1(doct)

for _, arg := range call.Arguments[1:] {
jp, err := c.jsonExtractPath(arg)
if err != nil {
return ctype{}, err
}
paths = append(paths, jp)
// TODO: `*compiler.compileParseJSON` should handle `sqltypes.Null`` properly but
// we'll handle it here until all call sites are fixed.
var jt ctype
if doct.Type != sqltypes.Null {
jt, err = c.compileParseJSON("JSON_EXTRACT", doct, 1)
if err != nil {
return ctype{}, err
}
} else {
jt = ctype{Type: sqltypes.Null, Flag: flagNull | flagNullable, Col: collationNull}
}

staticPaths := make([]staticPath, 0, len(call.Arguments[1:]))

jt, err := c.compileParseJSON("JSON_EXTRACT", doct, 1)
for _, arg := range call.Arguments[1:] {
argType, err := arg.compile(c)
if err != nil {
return ctype{}, err
}

c.asm.Fn_JSON_EXTRACT0(paths)
return jt, nil
if !nullable {
nullable = argType.nullable()
}

if arg.constant() {
staticEnv := EmptyExpressionEnv(c.env)
arg, err = simplifyExpr(staticEnv, arg)
if err != nil {
return ctype{}, err
}

p, err := c.jsonExtractPath(arg)
staticPaths = append(staticPaths, staticPath{p, err})
} else {
staticPaths = append(staticPaths, staticPath{nil, nil})
}
}

return ctype{}, c.unsupported(call)
c.asm.Fn_JSON_EXTRACT(len(call.Arguments[1:]), staticPaths)
c.asm.jumpDestination(skip)

if nullable {
// If any argument is nullable, the result is nullable too
jt.Flag |= flagNullable
}

return jt, nil
}

func (call *builtinJSONUnquote) eval(env *ExpressionEnv) (eval, error) {
Expand Down
1 change: 1 addition & 0 deletions go/vt/vtgate/evalengine/integration/fuzz_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ var (
regexp.MustCompile(`Illegal argument to a regular expression`),
regexp.MustCompile(`Incorrect arguments to regexp_substr`),
regexp.MustCompile(`Incorrect arguments to regexp_replace`),
regexp.MustCompile(`Invalid JSON path expression\. The error is around character position (\d+)\.`),
Copy link
Copy Markdown
Member Author

@arthurschreiber arthurschreiber Dec 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's currently some differences in the error messages generated by the evalengine vs what MySQL generates when JSON Path parsing runs into issues.

For now, I'd like to ignore these differences - but I do believe we should be able to change our implementation in the future to mimic MySQL more closely.

}
)

Expand Down
59 changes: 55 additions & 4 deletions go/vt/vtgate/evalengine/testcases/cases.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ import (

var Cases = []TestCase{
{Run: JSONExtract, Schema: JSONExtract_Schema},
{Run: JSONPathOperations},
{Run: FnJSONKeys},
{Run: FnJSONExtract},
{Run: FnJSONContainsPath},
{Run: JSONArray},
{Run: JSONObject},
{Run: CharsetConversionOperators},
Expand Down Expand Up @@ -176,18 +178,67 @@ var Cases = []TestCase{
{Run: RegexpReplace},
}

func JSONPathOperations(yield Query) {
func FnJSONKeys(yield Query) {
for _, obj := range inputJSONObjects {
yield(fmt.Sprintf("JSON_KEYS('%s')", obj), nil, false)

for _, path1 := range inputJSONPaths {
yield(fmt.Sprintf("JSON_KEYS('%s', '%s')", obj, path1), nil, false)
}
}
}

func FnJSONExtract(yield Query) {
for _, obj := range inputJSONObjects {
for _, path1 := range inputJSONPaths {
yield(fmt.Sprintf("JSON_EXTRACT('%s', '%s')", obj, path1), nil, false)

for _, path2 := range inputJSONPaths {
yield(fmt.Sprintf("JSON_EXTRACT('%s', '%s', '%s')", obj, path1, path2), nil, false)
}
}
}

yield(`JSON_EXTRACT('{"a": 1}', '$.a')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', '$.*')`, nil, false)
yield(`JSON_EXTRACT('[1, 2, 3]', '$[0 to 2]')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1, "b": 2}', '$.a', '$.b')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', '$.a', '$.z')`, nil, false)

yield(`JSON_EXTRACT(CONCAT('{', '"a"', ':', ' ', '1', '}'), '$.a')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', CONCAT('$', '.', 'a'))`, nil, false)

yield(`JSON_EXTRACT(NULL, '$.a')`, nil, false)
yield(`JSON_EXTRACT(NULL, NULL)`, nil, false)

yield(`JSON_EXTRACT('{"a": 1}', NULL)`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', '$.a', NULL)`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', NULL, '$.a')`, nil, false)

yield(`JSON_EXTRACT('{"a": 1}', '$.b')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', '$.b', '$.c')`, nil, false)
yield(`JSON_EXTRACT('[1,2,3]', '$[10]')`, nil, false)

yield(`JSON_EXTRACT('{invalid}', '$.a')`, nil, false)
yield(`JSON_EXTRACT('not json', '$.a')`, nil, false)
yield(`JSON_EXTRACT('', '$.a')`, nil, false)
yield(`JSON_EXTRACT('{invalid}', NULL)`, nil, false)

yield(`JSON_EXTRACT('{"a": 1}', '$.b[ 1 ].')`, nil, false)

yield(`JSON_EXTRACT(NULL, 'invalid-path')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', NULL, 'invalid-path')`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', 'invalid-path', NULL)`, nil, false)
yield(`JSON_EXTRACT('{"a": 1}', '$.a', 'invalid')`, nil, false)
Comment on lines +202 to +232
Copy link
Copy Markdown
Member Author

@arthurschreiber arthurschreiber Dec 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some of these could be handled by adding them to inputJSONObjects or inputJSONPaths, but that then highlights bugs in JSON_KEYS and JSON_CONTAINS, which I want to focus on in separate PRs.

}

func FnJSONContainsPath(yield Query) {
for _, obj := range inputJSONObjects {
for _, path1 := range inputJSONPaths {
yield(fmt.Sprintf("JSON_CONTAINS_PATH('%s', 'one', '%s')", obj, path1), nil, false)
yield(fmt.Sprintf("JSON_CONTAINS_PATH('%s', 'all', '%s')", obj, path1), nil, false)
yield(fmt.Sprintf("JSON_KEYS('%s', '%s')", obj, path1), nil, false)

for _, path2 := range inputJSONPaths {
yield(fmt.Sprintf("JSON_EXTRACT('%s', '%s', '%s')", obj, path1, path2), nil, false)
yield(fmt.Sprintf("JSON_CONTAINS_PATH('%s', 'one', '%s', '%s')", obj, path1, path2), nil, false)
yield(fmt.Sprintf("JSON_CONTAINS_PATH('%s', 'all', '%s', '%s')", obj, path1, path2), nil, false)
}
Expand Down
Loading