Skip to content

Commit 3a7905e

Browse files
mtodormtodor
committed
Add tests
1 parent 3fd2498 commit 3a7905e

File tree

3 files changed

+238
-0
lines changed

3 files changed

+238
-0
lines changed

‎cmd/flakechecker/flake_config_test.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package main
2+
3+
import (
4+
"github.com/stretchr/testify/assert"
5+
"testing"
6+
)
7+
8+
func TestLoadFlakeConfigFile(t *testing.T) {
9+
samples := []struct {
10+
name string
11+
fileName string
12+
13+
expectError bool
14+
expectConfig []*flakeDetectionPolicy
15+
}{
16+
{
17+
name: "no config file",
18+
fileName: "no_config.yml",
19+
expectError: true,
20+
expectConfig: nil,
21+
},
22+
{
23+
name: "valid config file",
24+
fileName: "testdata/flake-config.yml",
25+
expectError: false,
26+
expectConfig: []*flakeDetectionPolicy{
27+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{
28+
MatchJobName: "pr-.*",
29+
RatioJobName: "main-branch-tests",
30+
TestNameRegex: "TestLoadFlakeConf.*",
31+
Classname: "TestLoadFlakeConfigFile",
32+
RatioThreshold: 5,
33+
}),
34+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{
35+
MatchJobName: "pull-request-tests",
36+
RatioJobName: "main-branch-tests",
37+
TestNameRegex: "TestLoadFlakeConfigFile",
38+
Classname: "TestLoadFlakeConfigFile",
39+
RatioThreshold: 10,
40+
}),
41+
},
42+
},
43+
}
44+
45+
for _, sample := range samples {
46+
t.Run(sample.name, func(tt *testing.T) {
47+
config, err := loadFlakeConfigFile(sample.fileName)
48+
49+
assert.Equal(tt, sample.expectError, err != nil)
50+
assert.Equal(tt, sample.expectConfig, config)
51+
})
52+
}
53+
}

‎cmd/flakechecker/main_test.go

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
package main
2+
3+
import (
4+
"github.com/pkg/errors"
5+
"github.com/stackrox/junit2jira/pkg/testcase"
6+
"github.com/stretchr/testify/assert"
7+
"testing"
8+
)
9+
10+
type biqQueryClientMock struct {
11+
getRatioForTest func(flakeTestConfig *flakeDetectionPolicy, testName string) (int, int, error)
12+
}
13+
14+
func (c *biqQueryClientMock) GetRatioForTest(flakeTestConfig *flakeDetectionPolicy, testName string) (int, int, error) {
15+
if c.getRatioForTest != nil {
16+
return c.getRatioForTest(flakeTestConfig, testName)
17+
}
18+
19+
// By default, fail. In most cases, we will not reach this part in tests.
20+
return 0, 0, errors.New("fail")
21+
}
22+
23+
func getRatioForTestNoFailures(_ *flakeDetectionPolicy, _ string) (int, int, error) {
24+
return totalRunsLimit, 0, nil
25+
}
26+
27+
func getRatioForTestAllFailures(_ *flakeDetectionPolicy, _ string) (int, int, error) {
28+
return totalRunsLimit, 50, nil
29+
}
30+
31+
func TestCheckFailedTests(t *testing.T) {
32+
p := flakeCheckerParams{jobName: "test-job"}
33+
34+
samples := map[string]struct {
35+
bqClient biqQueryClient
36+
failedTests []testcase.TestCase
37+
flakeDetectionPolicies []*flakeDetectionPolicy
38+
expectError bool
39+
expectErrorStr string
40+
}{
41+
"no failed tests": {
42+
bqClient: &biqQueryClientMock{},
43+
failedTests: []testcase.TestCase{},
44+
flakeDetectionPolicies: []*flakeDetectionPolicy{
45+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job1-", TestNameRegex: "test name"}),
46+
},
47+
expectError: true,
48+
expectErrorStr: errDescNoFailedTests,
49+
},
50+
"no config match - job name": {
51+
bqClient: &biqQueryClientMock{
52+
getRatioForTest: getRatioForTestNoFailures,
53+
},
54+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
55+
flakeDetectionPolicies: []*flakeDetectionPolicy{
56+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job-1", TestNameRegex: "test", Classname: "class"}),
57+
},
58+
expectError: true,
59+
expectErrorStr: errDescNoMatch,
60+
},
61+
"no config match - test name": {
62+
bqClient: &biqQueryClientMock{
63+
getRatioForTest: getRatioForTestNoFailures,
64+
},
65+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
66+
flakeDetectionPolicies: []*flakeDetectionPolicy{
67+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "wrong-test", Classname: "class"}),
68+
},
69+
expectError: true,
70+
expectErrorStr: errDescNoMatch,
71+
},
72+
"no config match - classname": {
73+
bqClient: &biqQueryClientMock{
74+
getRatioForTest: getRatioForTestNoFailures,
75+
},
76+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
77+
flakeDetectionPolicies: []*flakeDetectionPolicy{
78+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "wrong-class"}),
79+
},
80+
expectError: true,
81+
expectErrorStr: errDescNoMatch,
82+
},
83+
"unable to fetch ratio": {
84+
bqClient: &biqQueryClientMock{
85+
getRatioForTest: func(_ *flakeDetectionPolicy, _ string) (int, int, error) {
86+
return 0, 0, errors.New("fail")
87+
},
88+
},
89+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
90+
flakeDetectionPolicies: []*flakeDetectionPolicy{
91+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
92+
},
93+
expectError: true,
94+
expectErrorStr: errDescGetRatio,
95+
},
96+
"total runs below limit": {
97+
bqClient: &biqQueryClientMock{
98+
getRatioForTest: func(_ *flakeDetectionPolicy, _ string) (int, int, error) {
99+
return totalRunsLimit - 1, 0, nil
100+
},
101+
},
102+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
103+
flakeDetectionPolicies: []*flakeDetectionPolicy{
104+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
105+
},
106+
expectError: true,
107+
expectErrorStr: errDescShortHistory,
108+
},
109+
"fail ratio below threshold": {
110+
bqClient: &biqQueryClientMock{
111+
getRatioForTest: getRatioForTestNoFailures,
112+
},
113+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
114+
flakeDetectionPolicies: []*flakeDetectionPolicy{
115+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
116+
},
117+
expectError: false,
118+
},
119+
"fail ratio above threshold": {
120+
bqClient: &biqQueryClientMock{
121+
getRatioForTest: getRatioForTestAllFailures,
122+
},
123+
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
124+
flakeDetectionPolicies: []*flakeDetectionPolicy{
125+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
126+
},
127+
expectError: true,
128+
expectErrorStr: errDescAboveThreshold,
129+
},
130+
"fail ratio below threshold - multiple tests": {
131+
bqClient: &biqQueryClientMock{
132+
getRatioForTest: getRatioForTestNoFailures,
133+
},
134+
failedTests: []testcase.TestCase{
135+
{Name: "test", Classname: "class"},
136+
{Name: "test-1", Classname: "class-1"},
137+
},
138+
flakeDetectionPolicies: []*flakeDetectionPolicy{
139+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
140+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-1", Classname: "class-1", RatioThreshold: 1}),
141+
},
142+
expectError: false,
143+
},
144+
"fail ratio above threshold - multiple tests": {
145+
bqClient: &biqQueryClientMock{
146+
getRatioForTest: getRatioForTestAllFailures,
147+
},
148+
failedTests: []testcase.TestCase{
149+
{Name: "test-ratio-below", Classname: "class"},
150+
{Name: "test-ratio-above", Classname: "class"},
151+
},
152+
flakeDetectionPolicies: []*flakeDetectionPolicy{
153+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-below", Classname: "class", RatioThreshold: 90}),
154+
newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-above", Classname: "class", RatioThreshold: 10}),
155+
},
156+
expectError: true,
157+
expectErrorStr: errDescAboveThreshold,
158+
},
159+
}
160+
161+
for sampleName, sample := range samples {
162+
t.Run(sampleName, func(tt *testing.T) {
163+
err := p.checkFailedTests(sample.bqClient, sample.failedTests, sample.flakeDetectionPolicies)
164+
165+
if sample.expectError {
166+
assert.ErrorContains(tt, err, sample.expectErrorStr)
167+
} else {
168+
assert.NoError(tt, err)
169+
}
170+
})
171+
}
172+
}

‎cmd/flakechecker/testdata/flake-config.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Config with regex
2+
- matchJobName: 'pr-.*'
3+
ratioJobName: main-branch-tests
4+
testNameRegex: 'TestLoadFlakeConf.*'
5+
classname: TestLoadFlakeConfigFile
6+
ratioThreshold: 5
7+
8+
# Config with flat values
9+
- matchJobName: pull-request-tests
10+
ratioJobName: main-branch-tests
11+
testNameRegex: TestLoadFlakeConfigFile
12+
classname: TestLoadFlakeConfigFile
13+
ratioThreshold: 10

0 commit comments

Comments
 (0)