Skip to content

Commit 6d61779

Browse files
bwplotkabwplotka
committed
Trying to speed up tests a bit.
Signed-off-by: Bartek Plotka <bwplotka@gmail.com>
1 parent f4ec30e commit 6d61779

File tree

7 files changed

+66
-80
lines changed

7 files changed

+66
-80
lines changed

‎pkg/objstore/azure/azure.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err
291291
conf := &Config{
292292
StorageAccountName: os.Getenv("AZURE_STORAGE_ACCOUNT"),
293293
StorageAccountKey: os.Getenv("AZURE_STORAGE_ACCESS_KEY"),
294-
ContainerName: "thanos-e2e-test",
294+
ContainerName: objstore.CreateTemporaryTestBucketName(t),
295295
}
296296

297297
bc, err := yaml.Marshal(conf)

‎pkg/objstore/cos/cos.go‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,17 @@ import (
44
"context"
55
"fmt"
66
"io"
7-
"math/rand"
87
"net/http"
98
"os"
109
"strings"
1110
"testing"
12-
"time"
1311

1412
"github.com/go-kit/kit/log"
15-
cos "github.com/mozillazg/go-cos"
13+
"github.com/mozillazg/go-cos"
1614
"github.com/pkg/errors"
1715
"github.com/thanos-io/thanos/pkg/objstore"
1816
"github.com/thanos-io/thanos/pkg/runutil"
19-
yaml "gopkg.in/yaml.v2"
17+
"gopkg.in/yaml.v2"
2018
)
2119

2220
// DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -311,14 +309,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
311309
t.Log("WARNING. Reusing", c.Bucket, "COS bucket for COS tests. Manual cleanup afterwards is required")
312310
return b, func() {}, nil
313311
}
314-
315-
src := rand.NewSource(time.Now().UnixNano())
316-
317-
tmpBucketName := strings.Replace(fmt.Sprintf("test_%x", src.Int63()), "_", "-", -1)
318-
if len(tmpBucketName) >= 31 {
319-
tmpBucketName = tmpBucketName[:31]
320-
}
321-
c.Bucket = tmpBucketName
312+
c.Bucket = objstore.CreateTemporaryTestBucketName(t)
322313

323314
bc, err := yaml.Marshal(c)
324315
if err != nil {
@@ -333,12 +324,12 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
333324
if _, err := b.client.Bucket.Put(context.Background(), nil); err != nil {
334325
return nil, nil, err
335326
}
336-
t.Log("created temporary COS bucket for COS tests with name", tmpBucketName)
327+
t.Log("created temporary COS bucket for COS tests with name", c.Bucket)
337328

338329
return b, func() {
339330
objstore.EmptyBucket(t, context.Background(), b)
340331
if _, err := b.client.Bucket.Delete(context.Background()); err != nil {
341-
t.Logf("deleting bucket %s failed: %s", tmpBucketName, err)
332+
t.Logf("deleting bucket %s failed: %s", c.Bucket, err)
342333
}
343334
}, nil
344335
}

‎pkg/objstore/gcs/gcs.go‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@ import (
55
"context"
66
"fmt"
77
"io"
8-
"math/rand"
98
"runtime"
109
"strings"
1110
"testing"
12-
"time"
1311

1412
"cloud.google.com/go/storage"
1513
"github.com/go-kit/kit/log"
@@ -19,7 +17,7 @@ import (
1917
"golang.org/x/oauth2/google"
2018
"google.golang.org/api/iterator"
2119
"google.golang.org/api/option"
22-
yaml "gopkg.in/yaml.v2"
20+
"gopkg.in/yaml.v2"
2321
)
2422

2523
// DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -169,9 +167,8 @@ func (b *Bucket) Close() error {
169167
func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error) {
170168
ctx, cancel := context.WithCancel(context.Background())
171169
defer cancel()
172-
src := rand.NewSource(time.Now().UnixNano())
173170
gTestConfig := Config{
174-
Bucket: fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()),
171+
Bucket: objstore.CreateTemporaryTestBucketName(t),
175172
}
176173

177174
bc, err := yaml.Marshal(gTestConfig)

‎pkg/objstore/objstore.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@ package objstore
22

33
import (
44
"context"
5+
"fmt"
56
"io"
7+
"math/rand"
68
"os"
79
"path/filepath"
810
"strings"
11+
"testing"
912
"time"
1013

1114
"github.com/go-kit/kit/log"
@@ -372,3 +375,14 @@ func (rc *timingReadCloser) Read(b []byte) (n int, err error) {
372375
}
373376
return n, err
374377
}
378+
379+
func CreateTemporaryTestBucketName(t testing.TB) string {
380+
src := rand.NewSource(time.Now().UnixNano())
381+
382+
// Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html.
383+
name := strings.Replace(strings.Replace(fmt.Sprintf("test_%x_%s", src.Int63(), strings.ToLower(t.Name())), "_", "-", -1), "/", "-", -1)
384+
if len(name) >= 63 {
385+
name = name[:63]
386+
}
387+
return name
388+
}

‎pkg/objstore/objtesting/foreach.go‎

Lines changed: 39 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ package objtesting
33
import (
44
"os"
55
"testing"
6-
"time"
76

8-
"github.com/fortytw2/leaktest"
97
"github.com/thanos-io/thanos/pkg/objstore"
108
"github.com/thanos-io/thanos/pkg/objstore/azure"
119
"github.com/thanos-io/thanos/pkg/objstore/cos"
@@ -21,98 +19,98 @@ import (
2119
// that deletes it after test was run.
2220
// Use THANOS_SKIP_<objstorename>_TESTS to skip explicitly certain tests.
2321
func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) {
22+
t.Parallel()
23+
2424
// Mandatory Inmem.
2525
if ok := t.Run("inmem", func(t *testing.T) {
26-
defer leaktest.CheckTimeout(t, 10*time.Second)()
27-
26+
t.Parallel()
2827
testFn(t, inmem.NewBucket())
29-
3028
}); !ok {
3129
return
3230
}
3331

3432
// Optional GCS.
3533
if _, ok := os.LookupEnv("THANOS_SKIP_GCS_TESTS"); !ok {
36-
bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
37-
testutil.Ok(t, err)
34+
t.Run("gcs", func(t *testing.T) {
35+
bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
36+
testutil.Ok(t, err)
37+
38+
t.Parallel()
39+
defer closeFn()
3840

39-
ok := t.Run("gcs", func(t *testing.T) {
4041
// TODO(bwplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved.
4142
testFn(t, bkt)
4243
})
43-
closeFn()
44-
if !ok {
45-
return
46-
}
44+
4745
} else {
4846
t.Log("THANOS_SKIP_GCS_TESTS envvar present. Skipping test against GCS.")
4947
}
5048

5149
// Optional S3.
5250
if _, ok := os.LookupEnv("THANOS_SKIP_S3_AWS_TESTS"); !ok {
53-
// TODO(bwplotka): Allow taking location from envvar.
54-
bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
55-
testutil.Ok(t, err)
51+
t.Run("aws s3", func(t *testing.T) {
52+
// TODO(bwplotka): Allow taking location from envvar.
53+
bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
54+
testutil.Ok(t, err)
55+
56+
t.Parallel()
57+
defer closeFn()
5658

57-
ok := t.Run("aws s3", func(t *testing.T) {
5859
// TODO(bwplotka): Add leaktest when we fix potential leak in minio library.
5960
// We cannot use leaktest for detecting our own potential leaks, when leaktest detects leaks in minio itself.
6061
// This needs to be investigated more.
6162

6263
testFn(t, bkt)
6364
})
64-
closeFn()
65-
if !ok {
66-
return
67-
}
65+
6866
} else {
6967
t.Log("THANOS_SKIP_S3_AWS_TESTS envvar present. Skipping test against S3 AWS.")
7068
}
7169

7270
// Optional Azure.
7371
if _, ok := os.LookupEnv("THANOS_SKIP_AZURE_TESTS"); !ok {
74-
bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
75-
testutil.Ok(t, err)
72+
t.Run("azure", func(t *testing.T) {
73+
bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
74+
testutil.Ok(t, err)
75+
76+
t.Parallel()
77+
defer closeFn()
7678

77-
ok := t.Run("azure", func(t *testing.T) {
7879
testFn(t, bkt)
7980
})
80-
closeFn()
81-
if !ok {
82-
return
83-
}
81+
8482
} else {
8583
t.Log("THANOS_SKIP_AZURE_TESTS envvar present. Skipping test against Azure.")
8684
}
8785

8886
// Optional SWIFT.
8987
if _, ok := os.LookupEnv("THANOS_SKIP_SWIFT_TESTS"); !ok {
90-
container, closeFn, err := swift.NewTestContainer(t)
91-
testutil.Ok(t, err)
88+
t.Run("swift", func(t *testing.T) {
89+
container, closeFn, err := swift.NewTestContainer(t)
90+
testutil.Ok(t, err)
91+
92+
t.Parallel()
93+
defer closeFn()
9294

93-
ok := t.Run("swift", func(t *testing.T) {
9495
testFn(t, container)
9596
})
96-
closeFn()
97-
if !ok {
98-
return
99-
}
97+
10098
} else {
10199
t.Log("THANOS_SKIP_SWIFT_TESTS envvar present. Skipping test against swift.")
102100
}
103101

104102
// Optional COS.
105103
if _, ok := os.LookupEnv("THANOS_SKIP_TENCENT_COS_TESTS"); !ok {
106-
bkt, closeFn, err := cos.NewTestBucket(t)
107-
testutil.Ok(t, err)
104+
t.Run("Tencent cos", func(t *testing.T) {
105+
bkt, closeFn, err := cos.NewTestBucket(t)
106+
testutil.Ok(t, err)
107+
108+
t.Parallel()
109+
defer closeFn()
108110

109-
ok := t.Run("Tencent cos", func(t *testing.T) {
110111
testFn(t, bkt)
111112
})
112-
closeFn()
113-
if !ok {
114-
return
115-
}
113+
116114
} else {
117115
t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.")
118116
}

‎pkg/objstore/s3/s3.go‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"crypto/tls"
77
"fmt"
88
"io"
9-
"math/rand"
109
"net"
1110
"net/http"
1211
"os"
@@ -18,15 +17,15 @@ import (
1817

1918
"github.com/go-kit/kit/log"
2019
"github.com/go-kit/kit/log/level"
21-
minio "github.com/minio/minio-go/v6"
20+
"github.com/minio/minio-go/v6"
2221
"github.com/minio/minio-go/v6/pkg/credentials"
2322
"github.com/minio/minio-go/v6/pkg/encrypt"
2423
"github.com/pkg/errors"
2524
"github.com/prometheus/common/model"
2625
"github.com/prometheus/common/version"
2726
"github.com/thanos-io/thanos/pkg/objstore"
2827
"github.com/thanos-io/thanos/pkg/runutil"
29-
yaml "gopkg.in/yaml.v2"
28+
"gopkg.in/yaml.v2"
3029
)
3130

3231
// DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -403,13 +402,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke
403402
}
404403

405404
if c.Bucket == "" {
406-
src := rand.NewSource(time.Now().UnixNano())
407-
408-
// Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html.
409-
bktToCreate = strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1)
410-
if len(bktToCreate) >= 63 {
411-
bktToCreate = bktToCreate[:63]
412-
}
405+
bktToCreate = objstore.CreateTemporaryTestBucketName(t)
413406
}
414407

415408
if err := b.client.MakeBucket(bktToCreate, location); err != nil {

‎pkg/objstore/swift/swift.go‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@ import (
55
"context"
66
"fmt"
77
"io"
8-
"math/rand"
98
"os"
109
"strings"
1110
"testing"
12-
"time"
1311

1412
"github.com/go-kit/kit/log"
1513
"github.com/gophercloud/gophercloud"
@@ -19,7 +17,7 @@ import (
1917
"github.com/gophercloud/gophercloud/pagination"
2018
"github.com/pkg/errors"
2119
"github.com/thanos-io/thanos/pkg/objstore"
22-
yaml "gopkg.in/yaml.v2"
20+
"gopkg.in/yaml.v2"
2321
)
2422

2523
// DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -291,12 +289,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) {
291289
return c, func() {}, nil
292290
}
293291

294-
src := rand.NewSource(time.Now().UnixNano())
295-
296-
tmpContainerName := fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63())
297-
if len(tmpContainerName) >= 63 {
298-
tmpContainerName = tmpContainerName[:63]
299-
}
292+
tmpContainerName := objstore.CreateTemporaryTestBucketName(t)
300293

301294
if err := c.createContainer(tmpContainerName); err != nil {
302295
return nil, nil, err

0 commit comments

Comments
 (0)