Skip to content

Commit ba4fd4c

Browse files
mtodormtodor
committed
Add tests
1 parent bd454ea commit ba4fd4c

File tree

3 files changed

+242
-0
lines changed

3 files changed

+242
-0
lines changed

‎cmd/flakechecker/flake_config_test.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package main
2+
3+
import (
4+
"github.com/stretchr/testify/assert"
5+
"testing"
6+
)
7+
8+
func TestLoadFlakeConfigFile(t *testing.T) {
9+
samples := []struct {
10+
name string
11+
fileName string
12+
13+
expectError bool
14+
expectConfig []*flakeDetectionPolicy
15+
}{
16+
{
17+
name: "no config file",
18+
fileName: "no_config.json",
19+
expectError: true,
20+
expectConfig: nil,
21+
},
22+
{
23+
name: "valid config file",
24+
fileName: "testdata/flake-config.json",
25+
expectError: false,
26+
expectConfig: []*flakeDetectionPolicy{
27+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{
28+
MatchJobName: "pr-.*",
29+
RatioJobName: "main-branch-tests",
30+
TestNameRegex: "TestLoadFlakeConf.*",
31+
Classname: "TestLoadFlakeConfigFile",
32+
RatioThreshold: 5,
33+
}),
34+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{
35+
MatchJobName: "pull-request-tests",
36+
RatioJobName: "main-branch-tests",
37+
TestNameRegex: "TestLoadFlakeConfigFile",
38+
Classname: "TestLoadFlakeConfigFile",
39+
RatioThreshold: 10,
40+
}),
41+
},
42+
},
43+
}
44+
45+
for _, sample := range samples {
46+
t.Run(sample.name, func(tt *testing.T) {
47+
config, err := loadFlakeConfigFile(sample.fileName)
48+
49+
assert.Equal(tt, sample.expectError, err != nil)
50+
assert.Equal(tt, sample.expectConfig, config)
51+
})
52+
}
53+
}

‎cmd/flakechecker/main_test.go

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
package main
2+
3+
import (
4+
"github.com/pkg/errors"
5+
"github.com/stackrox/junit2jira/pkg/testcase"
6+
"github.com/stretchr/testify/assert"
7+
"testing"
8+
)
9+
10+
type biqQueryClientMock struct {
11+
getRatioForTest func(flakeTestConfig *flakeDetectionPolicy, testName string) (int, int, error)
12+
}
13+
14+
func (c *biqQueryClientMock) GetRatioForTest(flakeTestConfig *flakeDetectionPolicy, testName string) (int, int, error) {
15+
if c.getRatioForTest != nil {
16+
return c.getRatioForTest(flakeTestConfig, testName)
17+
}
18+
19+
// By default, fail. In most cases, we will not reach this part in tests.
20+
return 0, 0, errors.New("fail")
21+
}
22+
23+
func getRatioForTestNoFailures(_ *flakeDetectionPolicy, _ string) (int, int, error) {
24+
return totalRunsLimit, 0, nil
25+
}
26+
27+
func getRatioForTestAllFailures(_ *flakeDetectionPolicy, _ string) (int, int, error) {
28+
return totalRunsLimit, 50, nil
29+
}
30+
31+
func TestCheckFailedTests(t *testing.T) {
32+
p := flakeCheckerParams{jobName: "test-job"}
33+
34+
samples := map[string]struct {
35+
bqClient biqQueryClient
36+
failedTests []testcase.TestCase
37+
flakeDetectionPolicies []*flakeDetectionPolicy
38+
expectError bool
39+
expectErrorStr string
40+
}{
41+
"no failed tests": {
42+
bqClient: &biqQueryClientMock{},
43+
failedTests: []testcase.TestCase{},
44+
flakeDetectionPolicies: []*flakeDetectionPolicy{
45+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job1-", TestNameRegex: "test name"}),
46+
},
47+
expectError: false,
48+
},
49+
"no config match - job name": {
50+
bqClient: &biqQueryClientMock{
51+
getRatioForTest: getRatioForTestNoFailures,
52+
},
53+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
54+
flakeDetectionPolicies: []*flakeDetectionPolicy{
55+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job-1", TestNameRegex: "test", Classname: "class"}),
56+
},
57+
expectError: true,
58+
expectErrorStr: errDescNoMatch,
59+
},
60+
"no config match - test name": {
61+
bqClient: &biqQueryClientMock{
62+
getRatioForTest: getRatioForTestNoFailures,
63+
},
64+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
65+
flakeDetectionPolicies: []*flakeDetectionPolicy{
66+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "wrong-test", Classname: "class"}),
67+
},
68+
expectError: true,
69+
expectErrorStr: errDescNoMatch,
70+
},
71+
"no config match - classname": {
72+
bqClient: &biqQueryClientMock{
73+
getRatioForTest: getRatioForTestNoFailures,
74+
},
75+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
76+
flakeDetectionPolicies: []*flakeDetectionPolicy{
77+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "wrong-class"}),
78+
},
79+
expectError: true,
80+
expectErrorStr: errDescNoMatch,
81+
},
82+
"unable to fetch ratio": {
83+
bqClient: &biqQueryClientMock{
84+
getRatioForTest: func(_ *flakeDetectionPolicy, _ string) (int, int, error) {
85+
return 0, 0, errors.New("fail")
86+
},
87+
},
88+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
89+
flakeDetectionPolicies: []*flakeDetectionPolicy{
90+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
91+
},
92+
expectError: true,
93+
expectErrorStr: errDescGetRatio,
94+
},
95+
"total runs below limit": {
96+
bqClient: &biqQueryClientMock{
97+
getRatioForTest: func(_ *flakeDetectionPolicy, _ string) (int, int, error) {
98+
return totalRunsLimit - 1, 0, nil
99+
},
100+
},
101+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
102+
flakeDetectionPolicies: []*flakeDetectionPolicy{
103+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
104+
},
105+
expectError: true,
106+
expectErrorStr: errDescShortHistory,
107+
},
108+
"fail ratio below threshold": {
109+
bqClient: &biqQueryClientMock{
110+
getRatioForTest: getRatioForTestNoFailures,
111+
},
112+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
113+
flakeDetectionPolicies: []*flakeDetectionPolicy{
114+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
115+
},
116+
expectError: false,
117+
},
118+
"fail ratio above threshold": {
119+
bqClient: &biqQueryClientMock{
120+
getRatioForTest: getRatioForTestAllFailures,
121+
},
122+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
123+
flakeDetectionPolicies: []*flakeDetectionPolicy{
124+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
125+
},
126+
expectError: true,
127+
expectErrorStr: errDescAboveThreshold,
128+
},
129+
"fail ratio below threshold - multiple tests": {
130+
bqClient: &biqQueryClientMock{
131+
getRatioForTest: getRatioForTestNoFailures,
132+
},
133+
failedTests: []testcase.TestCase{
134+
{Name: "test", Classname: "class"},
135+
{Name: "test-1", Classname: "class-1"},
136+
},
137+
flakeDetectionPolicies: []*flakeDetectionPolicy{
138+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
139+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-1", Classname: "class-1", RatioThreshold: 1}),
140+
},
141+
expectError: false,
142+
},
143+
"fail ratio above threshold - multiple tests": {
144+
bqClient: &biqQueryClientMock{
145+
getRatioForTest: getRatioForTestAllFailures,
146+
},
147+
failedTests: []testcase.TestCase{
148+
{Name: "test-ratio-below", Classname: "class"},
149+
{Name: "test-ratio-above", Classname: "class"},
150+
},
151+
flakeDetectionPolicies: []*flakeDetectionPolicy{
152+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-below", Classname: "class", RatioThreshold: 90}),
153+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-above", Classname: "class", RatioThreshold: 10}),
154+
},
155+
expectError: true,
156+
expectErrorStr: errDescAboveThreshold,
157+
},
158+
}
159+
160+
for sampleName, sample := range samples {
161+
t.Run(sampleName, func(tt *testing.T) {
162+
err := p.checkFailedTests(sample.bqClient, sample.failedTests, sample.flakeDetectionPolicies)
163+
164+
if sample.expectError {
165+
assert.ErrorContains(tt, err, sample.expectErrorStr)
166+
} else {
167+
assert.NoError(tt, err)
168+
}
169+
})
170+
}
171+
}

‎cmd/flakechecker/testdata/flake-config.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{
3+
"_comment": "Config with regex",
4+
"match_job_name": "pr-.*",
5+
"ratio_job_name": "main-branch-tests",
6+
"test_name_regex": "TestLoadFlakeConf.*",
7+
"classname": "TestLoadFlakeConfigFile",
8+
"ratio_threshold": 5
9+
},
10+
{
11+
"_comment": "Config with flat values",
12+
"match_job_name": "pull-request-tests",
13+
"ratio_job_name": "main-branch-tests",
14+
"test_name_regex": "TestLoadFlakeConfigFile",
15+
"classname": "TestLoadFlakeConfigFile",
16+
"ratio_threshold": 10
17+
}
18+
]

0 commit comments

Comments
 (0)