Skip to content

Commit ac24266

Browse files
yeya242nick
authored and
metalmatze
committed
Fix query frontend regression on release v0.17.0 (thanos-io#3480)
* query-frontend: make POST-request to downstream url for labels and series api endpoints (thanos-io#3444) Signed-off-by: Alexander Tunik <2braven@gmail.com> * remove default response cache config Signed-off-by: Ben Ye <yb532204897@gmail.com> * ensure order when merging multiple responses Signed-off-by: Ben Ye <yb532204897@gmail.com> Co-authored-by: Alexander Tunik <2braven@gmail.com>
1 parent d616214 commit ac24266

File tree

7 files changed

+306
-27
lines changed

7 files changed

+306
-27
lines changed

‎cmd/thanos/query_frontend.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,10 @@ func registerQueryFrontend(app *extkingpin.App) {
5151
MaxBodySize: 10 * 1024 * 1024,
5252
},
5353
QueryRangeConfig: queryfrontend.QueryRangeConfig{
54-
Limits: &cortexvalidation.Limits{},
55-
ResultsCacheConfig: &queryrange.ResultsCacheConfig{},
54+
Limits: &cortexvalidation.Limits{},
5655
},
5756
LabelsConfig: queryfrontend.LabelsConfig{
58-
Limits: &cortexvalidation.Limits{},
59-
ResultsCacheConfig: &queryrange.ResultsCacheConfig{},
57+
Limits: &cortexvalidation.Limits{},
6058
},
6159
},
6260
}

‎pkg/queryfrontend/labels_codec.go‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,20 +49,21 @@ func NewThanosLabelsCodec(partialResponse bool, defaultMetadataTimeRange time.Du
4949
}
5050
}
5151

52+
// MergeResponse merges multiple responses into a single Response. It needs to dedup the responses and ensure the order.
5253
func (c labelsCodec) MergeResponse(responses ...queryrange.Response) (queryrange.Response, error) {
5354
if len(responses) == 0 {
55+
// Empty response for label_names, label_values and series API.
5456
return &ThanosLabelsResponse{
5557
Status: queryrange.StatusSuccess,
5658
Data: []string{},
5759
}, nil
5860
}
5961

60-
if len(responses) == 1 {
61-
return responses[0], nil
62-
}
63-
6462
switch responses[0].(type) {
6563
case *ThanosLabelsResponse:
64+
if len(responses) == 1 {
65+
return responses[0], nil
66+
}
6667
set := make(map[string]struct{})
6768

6869
for _, res := range responses {
@@ -85,23 +86,18 @@ func (c labelsCodec) MergeResponse(responses ...queryrange.Response) (queryrange
8586
case *ThanosSeriesResponse:
8687
seriesData := make([]labelpb.ZLabelSet, 0)
8788

88-
// seriesString is used in soring so we don't have to calculate the string of label sets again.
89-
seriesString := make([]string, 0)
9089
uniqueSeries := make(map[string]struct{})
9190
for _, res := range responses {
9291
for _, series := range res.(*ThanosSeriesResponse).Data {
9392
s := series.PromLabels().String()
9493
if _, ok := uniqueSeries[s]; !ok {
9594
seriesData = append(seriesData, series)
96-
seriesString = append(seriesString, s)
9795
uniqueSeries[s] = struct{}{}
9896
}
9997
}
10098
}
10199

102-
sort.Slice(seriesData, func(i, j int) bool {
103-
return seriesString[i] < seriesString[j]
104-
})
100+
sort.Sort(seriesData)
105101
return &ThanosSeriesResponse{
106102
Status: queryrange.StatusSuccess,
107103
Data: seriesData,

‎pkg/queryfrontend/labels_codec_test.go‎

Lines changed: 155 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ func TestLabelsCodec_EncodeRequest(t *testing.T) {
234234
},
235235
{
236236
name: "thanos labels values request",
237-
req: &ThanosLabelsRequest{Start: 123000, End: 456000, Path: "/api/v1/label/__name__/values"},
237+
req: &ThanosLabelsRequest{Start: 123000, End: 456000, Path: "/api/v1/label/__name__/values", Label: "__name__"},
238238
checkFunc: func(r *http.Request) bool {
239239
return r.URL.Query().Get(start) == startTime &&
240240
r.URL.Query().Get(end) == endTime &&
@@ -243,7 +243,7 @@ func TestLabelsCodec_EncodeRequest(t *testing.T) {
243243
},
244244
{
245245
name: "thanos labels values request, partial response set to true",
246-
req: &ThanosLabelsRequest{Start: 123000, End: 456000, Path: "/api/v1/label/__name__/values", PartialResponse: true},
246+
req: &ThanosLabelsRequest{Start: 123000, End: 456000, Path: "/api/v1/label/__name__/values", Label: "__name__", PartialResponse: true},
247247
checkFunc: func(r *http.Request) bool {
248248
return r.URL.Query().Get(start) == startTime &&
249249
r.URL.Query().Get(end) == endTime &&
@@ -313,12 +313,29 @@ func TestLabelsCodec_DecodeResponse(t *testing.T) {
313313
labelsData, err := json.Marshal(labelResponse)
314314
testutil.Ok(t, err)
315315

316+
labelResponseWithHeaders := &ThanosLabelsResponse{
317+
Status: "success",
318+
Data: []string{"__name__"},
319+
Headers: []*ResponseHeader{{Name: cacheControlHeader, Values: []string{noStoreValue}}},
320+
}
321+
labelsDataWithHeaders, err := json.Marshal(labelResponseWithHeaders)
322+
testutil.Ok(t, err)
323+
316324
seriesResponse := &ThanosSeriesResponse{
317325
Status: "success",
318326
Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}},
319327
}
320328
seriesData, err := json.Marshal(seriesResponse)
321329
testutil.Ok(t, err)
330+
331+
seriesResponseWithHeaders := &ThanosSeriesResponse{
332+
Status: "success",
333+
Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}},
334+
Headers: []*ResponseHeader{{Name: cacheControlHeader, Values: []string{noStoreValue}}},
335+
}
336+
seriesDataWithHeaders, err := json.Marshal(seriesResponseWithHeaders)
337+
testutil.Ok(t, err)
338+
322339
for _, tc := range []struct {
323340
name string
324341
expectedError error
@@ -344,12 +361,34 @@ func TestLabelsCodec_DecodeResponse(t *testing.T) {
344361
res: http.Response{StatusCode: 200, Body: ioutil.NopCloser(bytes.NewBuffer(labelsData))},
345362
expectedResponse: labelResponse,
346363
},
364+
{
365+
name: "thanos labels request with HTTP headers",
366+
req: &ThanosLabelsRequest{},
367+
res: http.Response{
368+
StatusCode: 200, Body: ioutil.NopCloser(bytes.NewBuffer(labelsDataWithHeaders)),
369+
Header: map[string][]string{
370+
cacheControlHeader: {noStoreValue},
371+
},
372+
},
373+
expectedResponse: labelResponseWithHeaders,
374+
},
347375
{
348376
name: "thanos series request",
349377
req: &ThanosSeriesRequest{},
350378
res: http.Response{StatusCode: 200, Body: ioutil.NopCloser(bytes.NewBuffer(seriesData))},
351379
expectedResponse: seriesResponse,
352380
},
381+
{
382+
name: "thanos series request with HTTP headers",
383+
req: &ThanosSeriesRequest{},
384+
res: http.Response{
385+
StatusCode: 200, Body: ioutil.NopCloser(bytes.NewBuffer(seriesDataWithHeaders)),
386+
Header: map[string][]string{
387+
cacheControlHeader: {noStoreValue},
388+
},
389+
},
390+
expectedResponse: seriesResponseWithHeaders,
391+
},
353392
} {
354393
t.Run(tc.name, func(t *testing.T) {
355394
// Default partial response value doesn't matter when encoding requests.
@@ -364,3 +403,117 @@ func TestLabelsCodec_DecodeResponse(t *testing.T) {
364403
})
365404
}
366405
}
406+
407+
func TestLabelsCodec_MergeResponse(t *testing.T) {
408+
for _, tc := range []struct {
409+
name string
410+
expectedError error
411+
responses []queryrange.Response
412+
expectedResponse queryrange.Response
413+
}{
414+
{
415+
name: "Prometheus range query response format, not valid",
416+
responses: []queryrange.Response{
417+
&queryrange.PrometheusResponse{Status: "success"},
418+
},
419+
expectedError: httpgrpc.Errorf(http.StatusInternalServerError, "invalid response format"),
420+
},
421+
{
422+
name: "Empty response",
423+
responses: nil,
424+
expectedResponse: &ThanosLabelsResponse{Status: queryrange.StatusSuccess, Data: []string{}},
425+
},
426+
{
427+
name: "One label response",
428+
responses: []queryrange.Response{
429+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9090", "localhost:9091"}},
430+
},
431+
expectedResponse: &ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9090", "localhost:9091"}},
432+
},
433+
{
434+
name: "One label response and two empty responses",
435+
responses: []queryrange.Response{
436+
&ThanosLabelsResponse{Status: queryrange.StatusSuccess, Data: []string{}},
437+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9090", "localhost:9091"}},
438+
&ThanosLabelsResponse{Status: queryrange.StatusSuccess, Data: []string{}},
439+
},
440+
expectedResponse: &ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9090", "localhost:9091"}},
441+
},
442+
{
443+
name: "Multiple duplicate label responses",
444+
responses: []queryrange.Response{
445+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9090", "localhost:9091"}},
446+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9091", "localhost:9092"}},
447+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9092", "localhost:9093"}},
448+
},
449+
expectedResponse: &ThanosLabelsResponse{Status: "success",
450+
Data: []string{"localhost:9090", "localhost:9091", "localhost:9092", "localhost:9093"}},
451+
},
452+
// This case shouldn't happen because the responses from Querier are sorted.
453+
{
454+
name: "Multiple unordered label responses",
455+
responses: []queryrange.Response{
456+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9093", "localhost:9092"}},
457+
&ThanosLabelsResponse{Status: "success", Data: []string{"localhost:9091", "localhost:9090"}},
458+
},
459+
expectedResponse: &ThanosLabelsResponse{Status: "success",
460+
Data: []string{"localhost:9090", "localhost:9091", "localhost:9092", "localhost:9093"}},
461+
},
462+
{
463+
name: "One series response",
464+
responses: []queryrange.Response{
465+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
466+
},
467+
expectedResponse: &ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
468+
},
469+
{
470+
name: "One series response and two empty responses",
471+
responses: []queryrange.Response{
472+
&ThanosSeriesResponse{Status: queryrange.StatusSuccess},
473+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
474+
&ThanosSeriesResponse{Status: queryrange.StatusSuccess},
475+
},
476+
expectedResponse: &ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
477+
},
478+
{
479+
name: "Multiple duplicate series responses",
480+
responses: []queryrange.Response{
481+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
482+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
483+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
484+
},
485+
expectedResponse: &ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}}}},
486+
},
487+
{
488+
name: "Multiple unordered series responses",
489+
responses: []queryrange.Response{
490+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{
491+
{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}},
492+
{Labels: []labelpb.ZLabel{{Name: "test", Value: "aaa"}, {Name: "instance", Value: "localhost:9090"}}},
493+
}},
494+
&ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{
495+
{Labels: []labelpb.ZLabel{{Name: "foo", Value: "aaa"}}},
496+
{Labels: []labelpb.ZLabel{{Name: "test", Value: "bbb"}, {Name: "instance", Value: "localhost:9091"}}},
497+
}},
498+
},
499+
expectedResponse: &ThanosSeriesResponse{Status: "success", Data: []labelpb.ZLabelSet{
500+
{Labels: []labelpb.ZLabel{{Name: "foo", Value: "aaa"}}},
501+
{Labels: []labelpb.ZLabel{{Name: "foo", Value: "bar"}}},
502+
{Labels: []labelpb.ZLabel{{Name: "test", Value: "aaa"}, {Name: "instance", Value: "localhost:9090"}}},
503+
{Labels: []labelpb.ZLabel{{Name: "test", Value: "bbb"}, {Name: "instance", Value: "localhost:9091"}}},
504+
}},
505+
},
506+
} {
507+
t.Run(tc.name, func(t *testing.T) {
508+
// Default partial response value doesn't matter when encoding requests.
509+
codec := NewThanosLabelsCodec(false, time.Hour*2)
510+
r, err := codec.MergeResponse(tc.responses...)
511+
if tc.expectedError != nil {
512+
testutil.Equals(t, err, tc.expectedError)
513+
} else {
514+
testutil.Ok(t, err)
515+
testutil.Equals(t, tc.expectedResponse, r)
516+
}
517+
})
518+
}
519+
}

‎pkg/store/labelpb/label.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,3 +295,27 @@ func DeepCopy(lbls []ZLabel) []ZLabel {
295295
}
296296
return ret
297297
}
298+
299+
// ZLabelSets is a sortable list of ZLabelSet. It assumes the label pairs in each ZLabelSet element are already sorted.
300+
type ZLabelSets []ZLabelSet
301+
302+
func (z ZLabelSets) Len() int { return len(z) }
303+
304+
func (z ZLabelSets) Swap(i, j int) { z[i], z[j] = z[j], z[i] }
305+
306+
func (z ZLabelSets) Less(i, j int) bool {
307+
l := 0
308+
r := 0
309+
var result int
310+
for l < z[i].Size() && r < z[j].Size() {
311+
result = z[i].Labels[l].Compare(z[j].Labels[r])
312+
if result == 0 {
313+
l++
314+
r++
315+
continue
316+
}
317+
return result < 0
318+
}
319+
320+
return l == z[i].Size()
321+
}

‎pkg/store/labelpb/label_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ package labelpb
55

66
import (
77
"fmt"
8+
"reflect"
9+
"sort"
810
"testing"
911

1012
"github.com/prometheus/prometheus/pkg/labels"
@@ -104,3 +106,90 @@ func BenchmarkZLabelsMarshalUnmarshal(b *testing.B) {
104106
}
105107
})
106108
}
109+
110+
func TestSortZLabelSets(t *testing.T) {
111+
expectedResult := ZLabelSets{
112+
{
113+
Labels: ZLabelsFromPromLabels(
114+
labels.FromMap(map[string]string{
115+
"__name__": "grpc_client_handled_total",
116+
"cluster": "test",
117+
"grpc_code": "OK",
118+
"grpc_method": "Info",
119+
}),
120+
),
121+
},
122+
{
123+
Labels: ZLabelsFromPromLabels(
124+
labels.FromMap(map[string]string{
125+
"__name__": "grpc_client_handled_total",
126+
"cluster": "test",
127+
"grpc_code": "OK",
128+
"grpc_method": "LabelNames",
129+
}),
130+
),
131+
},
132+
{
133+
Labels: ZLabelsFromPromLabels(
134+
labels.FromMap(map[string]string{
135+
"__name__": "grpc_server_handled_total",
136+
"cluster": "test",
137+
"grpc_code": "OK",
138+
"grpc_method": "Info",
139+
}),
140+
),
141+
},
142+
{
143+
Labels: ZLabelsFromPromLabels(
144+
labels.FromMap(map[string]string{
145+
"__name__": "up",
146+
"instance": "localhost:10908",
147+
}),
148+
),
149+
},
150+
}
151+
152+
list := ZLabelSets{
153+
{
154+
Labels: ZLabelsFromPromLabels(
155+
labels.FromMap(map[string]string{
156+
"__name__": "up",
157+
"instance": "localhost:10908",
158+
}),
159+
),
160+
},
161+
{
162+
Labels: ZLabelsFromPromLabels(
163+
labels.FromMap(map[string]string{
164+
"__name__": "grpc_server_handled_total",
165+
"cluster": "test",
166+
"grpc_code": "OK",
167+
"grpc_method": "Info",
168+
}),
169+
),
170+
},
171+
{
172+
Labels: ZLabelsFromPromLabels(
173+
labels.FromMap(map[string]string{
174+
"__name__": "grpc_client_handled_total",
175+
"cluster": "test",
176+
"grpc_code": "OK",
177+
"grpc_method": "LabelNames",
178+
}),
179+
),
180+
},
181+
{
182+
Labels: ZLabelsFromPromLabels(
183+
labels.FromMap(map[string]string{
184+
"__name__": "grpc_client_handled_total",
185+
"cluster": "test",
186+
"grpc_code": "OK",
187+
"grpc_method": "Info",
188+
}),
189+
),
190+
},
191+
}
192+
193+
sort.Sort(list)
194+
reflect.DeepEqual(expectedResult, list)
195+
}

0 commit comments

Comments
 (0)