Skip to content

Commit 936722a

Browse files
committed
Merge pull request #1251 from youtube/proto
query.proto revamp
2 parents c5ab705 + 0e602ee commit 936722a

80 files changed

Lines changed: 3401 additions & 2602 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

go/cmd/vtcombo/tablet_map.go

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,10 @@ func (itc *internalTabletConn) Execute(ctx context.Context, query string, bindVa
193193
if err != nil {
194194
return nil, err
195195
}
196-
bindVars = tproto.Proto3ToBindVariables(bv)
196+
bindVars, err = tproto.Proto3ToBindVariables(bv)
197+
if err != nil {
198+
return nil, err
199+
}
197200
reply := &mproto.QueryResult{}
198201
if err := itc.tablet.qsc.QueryService().Execute(ctx, &pbq.Target{
199202
Keyspace: itc.tablet.keyspace,
@@ -218,8 +221,12 @@ func (itc *internalTabletConn) ExecuteBatch(ctx context.Context, queries []tprot
218221
if err != nil {
219222
return nil, err
220223
}
224+
bindVars, err := tproto.Proto3ToBindVariables(bv)
225+
if err != nil {
226+
return nil, err
227+
}
221228
q[i].Sql = query.Sql
222-
q[i].BindVariables = tproto.Proto3ToBindVariables(bv)
229+
q[i].BindVariables = bindVars
223230
}
224231
reply := &tproto.QueryResultList{}
225232
if err := itc.tablet.qsc.QueryService().ExecuteBatch(ctx, &pbq.Target{
@@ -243,7 +250,10 @@ func (itc *internalTabletConn) StreamExecute(ctx context.Context, query string,
243250
if err != nil {
244251
return nil, nil, err
245252
}
246-
bindVars = tproto.Proto3ToBindVariables(bv)
253+
bindVars, err = tproto.Proto3ToBindVariables(bv)
254+
if err != nil {
255+
return nil, nil, err
256+
}
247257
result := make(chan *mproto.QueryResult, 10)
248258
var finalErr error
249259

go/cmd/vtgateclienttest/goclienttest/echo.go

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"golang.org/x/net/context"
1111

12+
"github.com/youtube/vitess/go/sqltypes"
1213
"github.com/youtube/vitess/go/vt/callerid"
1314
"github.com/youtube/vitess/go/vt/key"
1415
"github.com/youtube/vitess/go/vt/vtgate/vtgateconn"
@@ -45,21 +46,21 @@ var (
4546
entityKeyspaceIDs = []*pbg.ExecuteEntityIdsRequest_EntityId{
4647
&pbg.ExecuteEntityIdsRequest_EntityId{
4748
KeyspaceId: []byte{1, 2, 3},
48-
XidType: pbg.ExecuteEntityIdsRequest_EntityId_TYPE_INT,
49-
XidInt: 123,
49+
XidType: sqltypes.Int64,
50+
XidValue: []byte("123"),
5051
},
5152
&pbg.ExecuteEntityIdsRequest_EntityId{
5253
KeyspaceId: []byte{4, 5, 6},
53-
XidType: pbg.ExecuteEntityIdsRequest_EntityId_TYPE_FLOAT,
54-
XidFloat: 2.0,
54+
XidType: sqltypes.Float64,
55+
XidValue: []byte("2"),
5556
},
5657
&pbg.ExecuteEntityIdsRequest_EntityId{
5758
KeyspaceId: []byte{7, 8, 9},
58-
XidType: pbg.ExecuteEntityIdsRequest_EntityId_TYPE_BYTES,
59-
XidBytes: []byte{1, 2, 3},
59+
XidType: sqltypes.VarBinary,
60+
XidValue: []byte{1, 2, 3},
6061
},
6162
}
62-
entityKeyspaceIDsEcho = "[xid_type:TYPE_INT xid_int:123 keyspace_id:\"\\001\\002\\003\" xid_type:TYPE_FLOAT xid_float:2 keyspace_id:\"\\004\\005\\006\" xid_type:TYPE_BYTES xid_bytes:\"\\001\\002\\003\" keyspace_id:\"\\007\\010\\t\" ]"
63+
entityKeyspaceIDsEcho = "[xid_type:INT64 xid_value:\"123\" keyspace_id:\"\\001\\002\\003\" xid_type:FLOAT64 xid_value:\"2\" keyspace_id:\"\\004\\005\\006\" xid_type:VARBINARY xid_value:\"\\001\\002\\003\" keyspace_id:\"\\007\\010\\t\" ]"
6364

6465
tabletType = pbt.TabletType_REPLICA
6566
tabletTypeEcho = pbt.TabletType_name[int32(tabletType)]
@@ -365,10 +366,10 @@ func testEchoSplitQuery(t *testing.T, conn *vtgateconn.VTGateConn) {
365366
}
366367

367368
// getEcho extracts the echoed field values from a query result.
368-
func getEcho(qr *mproto.QueryResult) map[string]string {
369-
values := map[string]string{}
369+
func getEcho(qr *mproto.QueryResult) map[string]sqltypes.Value {
370+
values := map[string]sqltypes.Value{}
370371
for i, field := range qr.Fields {
371-
values[field.Name] = qr.Rows[0][i].String()
372+
values[field.Name] = qr.Rows[0][i]
372373
}
373374
return values
374375
}
@@ -381,8 +382,16 @@ func checkEcho(t *testing.T, name string, qr *mproto.QueryResult, err error, wan
381382
}
382383
got := getEcho(qr)
383384
for k, v := range want {
384-
if got[k] != v {
385-
t.Errorf("%v: %v = %q, want %q", name, k, got[k], v)
385+
if got[k].String() != v {
386+
t.Errorf("%v: %v = \n%q, want \n%q", name, k, got[k], v)
386387
}
387388
}
389+
390+
// Check NULL and empty string.
391+
if !got["null"].IsNull() {
392+
t.Errorf("MySQL NULL value wasn't preserved")
393+
}
394+
if !got["emptyString"].IsString() || got["emptyString"].String() != "" {
395+
t.Errorf("Empty string value wasn't preserved")
396+
}
388397
}

go/cmd/vtgateclienttest/services/echo.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,17 @@ func echoQueryResult(vals map[string]interface{}) *mproto.QueryResult {
6464
qr := &mproto.QueryResult{}
6565

6666
var row []sqltypes.Value
67+
68+
// The first two returned fields are always a field with a MySQL NULL value,
69+
// and another field with a zero-length string.
70+
// Client tests can use this to check that they correctly distinguish the two.
71+
qr.Fields = append(qr.Fields, mproto.Field{Name: "null", Type: mproto.VT_VAR_STRING})
72+
row = append(row, sqltypes.NULL)
73+
qr.Fields = append(qr.Fields, mproto.Field{Name: "emptyString", Type: mproto.VT_VAR_STRING})
74+
row = append(row, sqltypes.MakeString(nil))
75+
6776
for k, v := range vals {
68-
qr.Fields = append(qr.Fields, mproto.Field{Name: k, Type: mproto.VT_VARCHAR})
77+
qr.Fields = append(qr.Fields, mproto.Field{Name: k, Type: mproto.VT_VAR_STRING})
6978

7079
val := reflect.ValueOf(v)
7180
if val.Kind() == reflect.Map {

go/mysql/proto/proto3.go

Lines changed: 48 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,20 +39,23 @@ func ProtoToCharset(c *pbb.Charset) *Charset {
3939
}
4040

4141
// FieldsToProto3 converts an internal []Field to the proto3 version
42-
func FieldsToProto3(f []Field) []*pbq.Field {
42+
func FieldsToProto3(f []Field) ([]*pbq.Field, error) {
4343
if len(f) == 0 {
44-
return nil
44+
return nil, nil
4545
}
4646

4747
result := make([]*pbq.Field, len(f))
4848
for i, f := range f {
49+
vitessType, err := sqltypes.MySQLToType(f.Type, f.Flags)
50+
if err != nil {
51+
return nil, err
52+
}
4953
result[i] = &pbq.Field{
50-
Name: f.Name,
51-
Type: pbq.Field_Type(f.Type),
52-
Flags: int64(f.Flags),
54+
Name: f.Name,
55+
Type: vitessType,
5356
}
5457
}
55-
return result
58+
return result, nil
5659
}
5760

5861
// Proto3ToFields converts a proto3 []Fields to an internal data structure.
@@ -63,8 +66,7 @@ func Proto3ToFields(f []*pbq.Field) []Field {
6366
result := make([]Field, len(f))
6467
for i, f := range f {
6568
result[i].Name = f.Name
66-
result[i].Type = int64(f.Type)
67-
result[i].Flags = int64(f.Flags)
69+
result[i].Type, result[i].Flags = sqltypes.TypeToMySQL(f.Type)
6870
}
6971
return result
7072
}
@@ -77,11 +79,25 @@ func RowsToProto3(rows [][]sqltypes.Value) []*pbq.Row {
7779

7880
result := make([]*pbq.Row, len(rows))
7981
for i, r := range rows {
80-
result[i] = &pbq.Row{
81-
Values: make([][]byte, len(r)),
82+
row := &pbq.Row{}
83+
result[i] = row
84+
row.Lengths = make([]int64, 0, len(r))
85+
total := 0
86+
for _, c := range r {
87+
if c.IsNull() {
88+
row.Lengths = append(row.Lengths, -1)
89+
continue
90+
}
91+
length := len(c.Raw())
92+
row.Lengths = append(row.Lengths, int64(length))
93+
total += length
8294
}
83-
for j, c := range r {
84-
result[i].Values[j] = c.Raw()
95+
row.Values = make([]byte, 0, total)
96+
for _, c := range r {
97+
if c.IsNull() {
98+
continue
99+
}
100+
row.Values = append(row.Values, c.Raw()...)
85101
}
86102
}
87103
return result
@@ -95,29 +111,41 @@ func Proto3ToRows(rows []*pbq.Row) [][]sqltypes.Value {
95111

96112
result := make([][]sqltypes.Value, len(rows))
97113
for i, r := range rows {
98-
result[i] = make([]sqltypes.Value, len(r.Values))
99-
for j, c := range r.Values {
100-
if c == nil {
114+
index := 0
115+
result[i] = make([]sqltypes.Value, len(r.Lengths))
116+
for j, l := range r.Lengths {
117+
if l <= -1 {
101118
result[i][j] = sqltypes.NULL
102119
} else {
103-
result[i][j] = sqltypes.MakeString(c)
120+
end := index + int(l)
121+
if end > len(r.Values) {
122+
result[i][j] = sqltypes.NULL
123+
index = len(r.Values)
124+
} else {
125+
result[i][j] = sqltypes.MakeString(r.Values[index:end])
126+
index = end
127+
}
104128
}
105129
}
106130
}
107131
return result
108132
}
109133

110134
// QueryResultToProto3 converts an internal QueryResult to the proto3 version
111-
func QueryResultToProto3(qr *QueryResult) *pbq.QueryResult {
135+
func QueryResultToProto3(qr *QueryResult) (*pbq.QueryResult, error) {
112136
if qr == nil {
113-
return nil
137+
return nil, nil
138+
}
139+
fields, err := FieldsToProto3(qr.Fields)
140+
if err != nil {
141+
return nil, err
114142
}
115143
return &pbq.QueryResult{
116-
Fields: FieldsToProto3(qr.Fields),
144+
Fields: fields,
117145
RowsAffected: qr.RowsAffected,
118146
InsertId: qr.InsertId,
119147
Rows: RowsToProto3(qr.Rows),
120-
}
148+
}, nil
121149
}
122150

123151
// Proto3ToQueryResult converts a proto3 QueryResult to an internal data structure.

go/mysql/proto/proto3_test.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Copyright 2015, Google Inc. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package proto
6+
7+
import (
8+
"reflect"
9+
"testing"
10+
11+
"github.com/youtube/vitess/go/sqltypes"
12+
"github.com/youtube/vitess/go/vt/proto/query"
13+
)
14+
15+
func TestFields(t *testing.T) {
16+
fields := []Field{{
17+
Name: "aa",
18+
Type: 1,
19+
Flags: 32,
20+
}, {
21+
Name: "bb",
22+
Type: 2,
23+
}}
24+
p3, err := FieldsToProto3(fields)
25+
if err != nil {
26+
t.Error(err)
27+
}
28+
wantp3 := []*query.Field{
29+
&query.Field{
30+
Name: "aa",
31+
Type: sqltypes.Uint8,
32+
},
33+
&query.Field{
34+
Name: "bb",
35+
Type: sqltypes.Int16,
36+
},
37+
}
38+
if !reflect.DeepEqual(p3, wantp3) {
39+
t.Errorf("p3: %v, want %v", p3, wantp3)
40+
}
41+
42+
reverse := Proto3ToFields(p3)
43+
if !reflect.DeepEqual(reverse, fields) {
44+
t.Errorf("reverse: %v, want %v", reverse, fields)
45+
}
46+
47+
fields = []Field{{
48+
Name: "aa",
49+
Type: 15,
50+
}}
51+
_, err = FieldsToProto3(fields)
52+
want := "Could not map: 15 to a vitess type"
53+
if err == nil || err.Error() != want {
54+
t.Errorf("Error: %v, want %v", err, want)
55+
}
56+
}
57+
58+
func TestRowsToProto3(t *testing.T) {
59+
rows := [][]sqltypes.Value{{
60+
sqltypes.MakeString([]byte("aa")),
61+
sqltypes.NULL,
62+
sqltypes.MakeString([]byte("12")),
63+
}, {
64+
sqltypes.MakeString([]byte("bb")),
65+
sqltypes.NULL,
66+
sqltypes.NULL,
67+
}}
68+
p3 := RowsToProto3(rows)
69+
want := []*query.Row{
70+
&query.Row{
71+
Lengths: []int64{2, -1, 2},
72+
Values: []byte("aa12"),
73+
},
74+
&query.Row{
75+
Lengths: []int64{2, -1, -1},
76+
Values: []byte("bb"),
77+
},
78+
}
79+
if !reflect.DeepEqual(p3, want) {
80+
t.Errorf("P3: %v, want %v", p3, want)
81+
}
82+
83+
reverse := Proto3ToRows(p3)
84+
if !reflect.DeepEqual(reverse, rows) {
85+
t.Errorf("reverse: \n%#v, want \n%#v", reverse, rows)
86+
}
87+
}
88+
89+
func TestInvalidRowsProto(t *testing.T) {
90+
p3 := []*query.Row{
91+
&query.Row{
92+
Lengths: []int64{3, 5, -1, 6},
93+
Values: []byte("aa12"),
94+
},
95+
}
96+
rows := Proto3ToRows(p3)
97+
want := [][]sqltypes.Value{{
98+
sqltypes.MakeString([]byte("aa1")),
99+
sqltypes.NULL,
100+
sqltypes.NULL,
101+
sqltypes.NULL,
102+
}}
103+
if !reflect.DeepEqual(rows, want) {
104+
t.Errorf("reverse: \n%#v, want \n%#v", rows, want)
105+
}
106+
107+
p3 = []*query.Row{
108+
&query.Row{
109+
Lengths: []int64{2, -2, 2},
110+
Values: []byte("aa12"),
111+
},
112+
}
113+
rows = Proto3ToRows(p3)
114+
want = [][]sqltypes.Value{{
115+
sqltypes.MakeString([]byte("aa")),
116+
sqltypes.NULL,
117+
sqltypes.MakeString([]byte("12")),
118+
}}
119+
if !reflect.DeepEqual(rows, want) {
120+
t.Errorf("reverse: \n%#v, want \n%#v", rows, want)
121+
}
122+
}

go/mysql/proto/structs.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ const (
2727
VT_DATETIME = 12
2828
VT_YEAR = 13
2929
VT_NEWDATE = 14
30-
VT_VARCHAR = 15
3130
VT_BIT = 16
3231
VT_NEWDECIMAL = 246
3332
VT_ENUM = 247

0 commit comments

Comments
 (0)