Skip to content

Commit 5cde2a2

Browse files
bwplotkabwplotka
committed
metadata: Fixed panic when no ext labels are set; Added more tests.
``` caller=fetcher.go:700 msg="block has no labels left, creating one" replica=deduped panic: assignment to entry in nil map goroutine 441 [running]: github.com/thanos-io/thanos/pkg/block.(*ReplicaLabelRemover).Modify(0xc000b19d70, 0x2b261e0, 0xc000c16d00, 0xc0008596e0, 0xc000b1a980, 0x0, 0x0) /remote-source/app/pkg/block/fetcher.go:701 +0x3cf github.com/thanos-io/thanos/pkg/block.(*BaseFetcher).fetch(0xc000d75d40, 0x2b261e0, 0xc000c16d00, 0xc000c16a40, 0xc000c16a00, 0x4, 0x4, 0xc000b0e590, 0x1, 0x1, ...) /remote-source/app/pkg/block/fetcher.go:441 +0x65d github.com/thanos-io/thanos/pkg/block.(*MetaFetcher).Fetch(0xc0002fe480, 0x2b261e0, 0xc000c16d00, 0x5528a0, 0xc000c16340, 0xc0005c6000, 0x8) /remote-source/app/pkg/block/fetcher.go:474 +0x9f github.com/thanos-io/thanos/pkg/compact.(*Syncer).SyncMetas(0xc000c67d80, 0x2b261e0, 0xc000c16d00, 0x0, 0x0) /remote-source/app/pkg/compact/compact.go:127 +0xc0 github.com/thanos-io/thanos/pkg/compact.(*BucketCompactor).Compact(0xc0002fed20, 0x2b261e0, 0xc000c16d00, 0x0, 0x0) /remote-source/app/pkg/compact/compact.go:945 +0x2c2 main.runCompact.func6(0xc000501c80, 0x0) /remote-source/app/cmd/thanos/compact.go:307 +0x15a main.runCompact.func7.1(0xc0006bbd70, 0xc000bbe0a0) /remote-source/app/cmd/thanos/compact.go:367 +0x99 github.com/thanos-io/thanos/pkg/runutil.Repeat(0x45d964b800, 0xc00005a660, 0xc0007e9f30, 0x0, 0x0) /remote-source/app/pkg/runutil/runutil.go:72 +0x91 main.runCompact.func7(0x0, 0x0) /remote-source/app/cmd/thanos/compact.go:366 +0x29a github.com/oklog/run.(*Group).Run.func1(0xc0001f9f20, 0xc000c67e00, 0xc000b0ecb0) /remote-source/deps/gomod/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x27 created by github.com/oklog/run.(*Group).Run /remote-source/deps/gomod/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xbb ``` Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
1 parent 755b275 commit 5cde2a2

File tree

15 files changed

+250
-20
lines changed

15 files changed

+250
-20
lines changed

‎cmd/thanos/receive.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ func registerReceive(app *extkingpin.App) {
101101
return errors.Wrap(err, "parse labels")
102102
}
103103

104+
if len(lset) == 0 {
105+
return errors.New("no external labels configured for receive, uniquely identifying external labels must be configured (ideally with `receive_` prefix); see https://thanos.io/tip/thanos/storage.md#external-labels for details.")
106+
}
107+
104108
var cw *receive.ConfigWatcher
105109
if *hashringsFile != "" {
106110
cw, err = receive.NewConfigWatcher(log.With(logger, "component", "config-watcher"), reg, *hashringsFile, *refreshInterval)

‎cmd/thanos/sidecar.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ func runSidecar(
166166
}
167167

168168
if len(m.Labels()) == 0 {
169-
return errors.New("no external labels configured on Prometheus server, uniquely identifying external labels must be configured")
169+
return errors.New("no external labels configured on Prometheus server, uniquely identifying external labels must be configured; see https://thanos.io/tip/thanos/storage.md#external-labels for details.")
170170
}
171171

172172
// Periodically query the Prometheus config. We use this as a heartbeat as well as for updating

‎pkg/block/block.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func Upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir st
8383
return errors.Wrap(err, "not a block dir")
8484
}
8585

86-
meta, err := metadata.Read(bdir)
86+
meta, err := metadata.ReadFromDir(bdir)
8787
if err != nil {
8888
// No meta or broken meta file.
8989
return errors.Wrap(err, "read meta")

‎pkg/block/fetcher.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ func (f *BaseFetcher) loadMeta(ctx context.Context, id ulid.ULID) (*metadata.Met
228228

229229
// Best effort load from local dir.
230230
if f.cacheDir != "" {
231-
m, err := metadata.Read(cachedBlockDir)
231+
m, err := metadata.ReadFromDir(cachedBlockDir)
232232
if err == nil {
233233
return m, nil
234234
}

‎pkg/block/index.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,7 @@ func Repair(logger log.Logger, dir string, id ulid.ULID, source metadata.SourceT
376376
entropy := rand.New(rand.NewSource(time.Now().UnixNano()))
377377
resid = ulid.MustNew(ulid.Now(), entropy)
378378

379-
meta, err := metadata.Read(bdir)
379+
meta, err := metadata.ReadFromDir(bdir)
380380
if err != nil {
381381
return resid, errors.Wrap(err, "read meta file")
382382
}

‎pkg/block/indexheader/header_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func TestReaders(t *testing.T) {
7676
db.Close()
7777
*/
7878

79-
m, err := metadata.Read("./testdata/index_format_v1")
79+
m, err := metadata.ReadFromDir("./testdata/index_format_v1")
8080
testutil.Ok(t, err)
8181
e2eutil.Copy(t, "./testdata/index_format_v1", filepath.Join(tmpDir, m.ULID.String()))
8282

@@ -312,7 +312,7 @@ func prepareIndexV2Block(t testing.TB, tmpDir string, bkt objstore.Bucket) *meta
312312
}
313313
*/
314314

315-
m, err := metadata.Read("./testdata/index_format_v2")
315+
m, err := metadata.ReadFromDir("./testdata/index_format_v2")
316316
testutil.Ok(t, err)
317317
e2eutil.Copy(t, "./testdata/index_format_v2", filepath.Join(tmpDir, m.ULID.String()))
318318

‎pkg/block/metadata/meta.go‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ package metadata
1111
import (
1212
"encoding/json"
1313
"io"
14-
"io/ioutil"
1514
"os"
1615
"path/filepath"
1716

@@ -57,6 +56,8 @@ type Thanos struct {
5756
// Version of Thanos meta file. If none specified, 1 is assumed (since first version did not have explicit version specified).
5857
Version int `json:"version,omitempty"`
5958

59+
// Labels are the external labels identifying the producer as well as tenant.
60+
// See https://thanos.io/tip/thanos/storage.md#external-labels for details.
6061
Labels map[string]string `json:"labels"`
6162
Downsample ThanosDownsample `json:"downsample"`
6263

@@ -87,7 +88,7 @@ type ThanosDownsample struct {
8788
// InjectThanos sets Thanos meta to the block meta JSON and saves it to the disk.
8889
// NOTE: It should be used after writing any block by any Thanos component, otherwise we will miss crucial metadata.
8990
func InjectThanos(logger log.Logger, bdir string, meta Thanos, downsampledMeta *tsdb.BlockMeta) (*Meta, error) {
90-
newMeta, err := Read(bdir)
91+
newMeta, err := ReadFromDir(bdir)
9192
if err != nil {
9293
return nil, errors.Wrap(err, "read new meta")
9394
}
@@ -154,17 +155,23 @@ func renameFile(logger log.Logger, from, to string) error {
154155
return pdir.Close()
155156
}
156157

157-
// Read reads the given meta from <dir>/meta.json.
158-
func Read(dir string) (*Meta, error) {
159-
b, err := ioutil.ReadFile(filepath.Join(dir, MetaFilename))
158+
// ReadFromDir reads the given meta from <dir>/meta.json.
159+
func ReadFromDir(dir string) (*Meta, error) {
160+
f, err := os.Open(filepath.Join(dir, MetaFilename))
160161
if err != nil {
161162
return nil, err
162163
}
163-
var m Meta
164+
return read(f)
165+
}
164166

165-
if err := json.Unmarshal(b, &m); err != nil {
167+
func read(rc io.ReadCloser) (_ *Meta, err error) {
168+
defer runutil.ExhaustCloseWithErrCapture(&err, rc, "close meta JSON")
169+
170+
var m Meta
171+
if err = json.NewDecoder(rc).Decode(&m); err != nil {
166172
return nil, err
167173
}
174+
168175
if m.Version != TSDBVersion1 {
169176
return nil, errors.Errorf("unexpected meta file version %d", m.Version)
170177
}
@@ -178,5 +185,10 @@ func Read(dir string) (*Meta, error) {
178185
if version != ThanosVersion1 {
179186
return nil, errors.Errorf("unexpected meta file Thanos section version %d", m.Version)
180187
}
188+
189+
if m.Thanos.Labels == nil {
190+
// To avoid extra nil checks, allocate map here if empty.
191+
m.Thanos.Labels = make(map[string]string)
192+
}
181193
return &m, nil
182194
}

‎pkg/block/metadata/meta_test.go‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
// Copyright (c) The Thanos Authors.
2+
// Licensed under the Apache License 2.0.
3+
4+
package metadata
5+
6+
import (
7+
"bytes"
8+
"io/ioutil"
9+
"testing"
10+
11+
"github.com/oklog/ulid"
12+
"github.com/prometheus/prometheus/tsdb"
13+
"github.com/thanos-io/thanos/pkg/testutil"
14+
)
15+
16+
func TestMeta_ReadWrite(t *testing.T) {
17+
t.Run("empty write/read/write", func(t *testing.T) {
18+
b := bytes.Buffer{}
19+
testutil.Ok(t, Meta{}.Write(&b))
20+
testutil.Equals(t, `{
21+
"ulid": "00000000000000000000000000",
22+
"minTime": 0,
23+
"maxTime": 0,
24+
"stats": {},
25+
"compaction": {
26+
"level": 0
27+
},
28+
"version": 0,
29+
"thanos": {
30+
"labels": null,
31+
"downsample": {
32+
"resolution": 0
33+
},
34+
"source": ""
35+
}
36+
}
37+
`, b.String())
38+
_, err := read(ioutil.NopCloser(&b))
39+
testutil.NotOk(t, err)
40+
testutil.Equals(t, "unexpected meta file version 0", err.Error())
41+
})
42+
43+
t.Run("real write/read/write", func(t *testing.T) {
44+
b := bytes.Buffer{}
45+
m1 := Meta{
46+
BlockMeta: tsdb.BlockMeta{
47+
ULID: ulid.MustNew(5, nil),
48+
MinTime: 2424,
49+
MaxTime: 134,
50+
Version: 1,
51+
Compaction: tsdb.BlockMetaCompaction{
52+
Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)},
53+
Parents: []tsdb.BlockDesc{
54+
{
55+
ULID: ulid.MustNew(3, nil),
56+
MinTime: 442,
57+
MaxTime: 24225,
58+
},
59+
},
60+
Level: 123,
61+
},
62+
Stats: tsdb.BlockStats{NumChunks: 14, NumSamples: 245, NumSeries: 4},
63+
},
64+
Thanos: Thanos{
65+
Version: 1,
66+
Labels: map[string]string{"ext": "lset1"},
67+
Source: ReceiveSource,
68+
Files: []File{
69+
{RelPath: "index", SizeBytes: 1313},
70+
},
71+
Downsample: ThanosDownsample{
72+
Resolution: 123144,
73+
},
74+
},
75+
}
76+
testutil.Ok(t, m1.Write(&b))
77+
testutil.Equals(t, `{
78+
"ulid": "00000000050000000000000000",
79+
"minTime": 2424,
80+
"maxTime": 134,
81+
"stats": {
82+
"numSamples": 245,
83+
"numSeries": 4,
84+
"numChunks": 14
85+
},
86+
"compaction": {
87+
"level": 123,
88+
"sources": [
89+
"00000000010000000000000000",
90+
"00000000020000000000000000"
91+
],
92+
"parents": [
93+
{
94+
"ulid": "00000000030000000000000000",
95+
"minTime": 442,
96+
"maxTime": 24225
97+
}
98+
]
99+
},
100+
"version": 1,
101+
"thanos": {
102+
"version": 1,
103+
"labels": {
104+
"ext": "lset1"
105+
},
106+
"downsample": {
107+
"resolution": 123144
108+
},
109+
"source": "receive",
110+
"files": [
111+
{
112+
"rel_path": "index",
113+
"size_bytes": 1313
114+
}
115+
]
116+
}
117+
}
118+
`, b.String())
119+
retMeta, err := read(ioutil.NopCloser(&b))
120+
testutil.Ok(t, err)
121+
testutil.Equals(t, m1, *retMeta)
122+
})
123+
124+
t.Run("missing external labels write/read/write", func(t *testing.T) {
125+
b := bytes.Buffer{}
126+
m1 := Meta{
127+
BlockMeta: tsdb.BlockMeta{
128+
ULID: ulid.MustNew(5, nil),
129+
MinTime: 2424,
130+
MaxTime: 134,
131+
Version: 1,
132+
Compaction: tsdb.BlockMetaCompaction{
133+
Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)},
134+
Parents: []tsdb.BlockDesc{
135+
{
136+
ULID: ulid.MustNew(3, nil),
137+
MinTime: 442,
138+
MaxTime: 24225,
139+
},
140+
},
141+
Level: 123,
142+
},
143+
Stats: tsdb.BlockStats{NumChunks: 14, NumSamples: 245, NumSeries: 4},
144+
},
145+
Thanos: Thanos{
146+
Version: 1,
147+
Source: ReceiveSource,
148+
Files: []File{
149+
{RelPath: "index", SizeBytes: 1313},
150+
},
151+
Downsample: ThanosDownsample{
152+
Resolution: 123144,
153+
},
154+
},
155+
}
156+
testutil.Ok(t, m1.Write(&b))
157+
testutil.Equals(t, `{
158+
"ulid": "00000000050000000000000000",
159+
"minTime": 2424,
160+
"maxTime": 134,
161+
"stats": {
162+
"numSamples": 245,
163+
"numSeries": 4,
164+
"numChunks": 14
165+
},
166+
"compaction": {
167+
"level": 123,
168+
"sources": [
169+
"00000000010000000000000000",
170+
"00000000020000000000000000"
171+
],
172+
"parents": [
173+
{
174+
"ulid": "00000000030000000000000000",
175+
"minTime": 442,
176+
"maxTime": 24225
177+
}
178+
]
179+
},
180+
"version": 1,
181+
"thanos": {
182+
"version": 1,
183+
"labels": null,
184+
"downsample": {
185+
"resolution": 123144
186+
},
187+
"source": "receive",
188+
"files": [
189+
{
190+
"rel_path": "index",
191+
"size_bytes": 1313
192+
}
193+
]
194+
}
195+
}
196+
`, b.String())
197+
retMeta, err := read(ioutil.NopCloser(&b))
198+
testutil.Ok(t, err)
199+
200+
// We expect map to be empty but allocated.
201+
testutil.Equals(t, map[string]string(nil), m1.Thanos.Labels)
202+
m1.Thanos.Labels = map[string]string{}
203+
testutil.Equals(t, m1, *retMeta)
204+
})
205+
}

‎pkg/compact/compact.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,7 @@ func RepairIssue347(ctx context.Context, logger log.Logger, bkt objstore.Bucket,
644644
return retry(errors.Wrapf(err, "download block %s", ie.id))
645645
}
646646

647-
meta, err := metadata.Read(bdir)
647+
meta, err := metadata.ReadFromDir(bdir)
648648
if err != nil {
649649
return errors.Wrapf(err, "read meta from %s", bdir)
650650
}

‎pkg/compact/compact_e2e_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) (
417417
}
418418
testutil.Ok(t, err)
419419

420-
meta, err := metadata.Read(filepath.Join(prepareDir, id.String()))
420+
meta, err := metadata.ReadFromDir(filepath.Join(prepareDir, id.String()))
421421
testutil.Ok(t, err)
422422
metas = append(metas, meta)
423423

0 commit comments

Comments
 (0)