Skip to content

Commit 54337ca

Browse files
EgorkaKulikovviktoriia-fomina
EgorkaKulikov
and
viktoriia-fomina
authored
Introduce utbot-framework-test project (#810)
* Introduce utbot-framework-test project * Add utbot-framework-test to actions * Fix compilation * `build-and-run-tests`: monitoring script location * Disabling test failing in CI run Co-authored-by: Victoria <[email protected]>
1 parent 309be9f commit 54337ca

File tree

253 files changed

+1075
-1115
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

253 files changed

+1075
-1115
lines changed

‎.github/workflows/build-and-run-tests-from-branch.yml

Lines changed: 75 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ jobs:
3030
steps:
3131
- name: Print environment variables
3232
run: printenv
33-
33+
3434
- name: Checkout repository
3535
uses: actions/checkout@v3
36-
36+
3737
- name: Check out ${{ github.event.inputs.commit_sha }} commit
3838
if: github.event.inputs.commit_sha != ''
3939
run: |
@@ -45,101 +45,102 @@ jobs:
4545
TASKS=$(echo $(cat .github/workflows/framework-tests-matrix.json))
4646
echo "::set-output name=matrix::$TASKS"
4747
echo $TASKS
48-
framework:
48+
49+
framework-test:
4950
# This job does not need to wait for 'prepare-tests-matrix' result.
50-
# GitHub allocates runners portionally. Framework tests are time consuming. That's why we want to force them
51+
# GitHub allocates runners portionally. Framework tests are time consuming. That's why we want to force them
5152
# to start execution early.
5253
needs: prepare-tests-matrix
53-
# Using matrices let create multiple jobs runs based on the combinations of the variables from matrices.
54+
# Using matrices let create multiple jobs runs based on the combinations of the variables from matrices.
5455
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
5556
strategy:
56-
# The option forces to execute all jobs even though some of them have failed.
57+
# The option forces to execute all jobs even though some of them have failed.
5758
fail-fast: false
5859
matrix: ${{ fromJson(needs.prepare-tests-matrix.outputs.matrix) }}
5960
runs-on: ubuntu-20.04
6061
container: unittestbot/java-env:java11-zulu-jdk-gradle7.4.2-kotlinc1.7.0
6162
steps:
62-
- name: Print environment variables
63-
run: printenv
64-
65-
- name: Checkout repository
66-
uses: actions/checkout@v3
63+
- name: Print environment variables
64+
run: printenv
6765

68-
- name: Check out ${{ github.event.inputs.commit_sha }} commit
69-
if: github.event.inputs.commit_sha != ''
70-
run: |
71-
git fetch
72-
git checkout ${{ github.event.inputs.commit_sha }}
73-
- name: Run monitoring
74-
run: |
75-
echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
76-
chmod +x ./scripts/project/monitoring.sh
77-
./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
78-
- name: Run tests
79-
run: |
80-
gradle --no-daemon :utbot-framework:test ${{ matrix.project.TESTS_TO_RUN }}
81-
- name: Upload logs
82-
if: ${{ always() }}
83-
uses: actions/upload-artifact@v3
84-
with:
85-
name: logs ${{ matrix.project.PART_NAME }}
86-
path: utbot-framework/logs/*
66+
- name: Checkout repository
67+
uses: actions/checkout@v3
68+
69+
- name: Check out ${{ github.event.inputs.commit_sha }} commit
70+
if: github.event.inputs.commit_sha != ''
71+
run: |
72+
git fetch
73+
git checkout ${{ github.event.inputs.commit_sha }}
74+
- name: Run monitoring
75+
run: |
76+
echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
77+
chmod +x ./scripts/project/monitoring.sh
78+
./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
79+
- name: Run tests
80+
run: |
81+
gradle --no-daemon :utbot-framework-test:test ${{ matrix.project.TESTS_TO_RUN }}
82+
- name: Upload logs
83+
if: ${{ always() }}
84+
uses: actions/upload-artifact@v3
85+
with:
86+
name: logs ${{ matrix.project.PART_NAME }}
87+
path: utbot-framework-test/logs/*
8788

88-
- name: Upload UTBot temp directory content
89-
if: ${{ always() }}
90-
uses: actions/upload-artifact@v3
91-
with:
92-
name: utbot_temp ${{ matrix.project.PART_NAME }}
93-
path: |
94-
/tmp/UTBot/generated*/*
95-
/tmp/UTBot/utbot-childprocess-errors/*
96-
- name: Upload test report if tests have failed
97-
if: ${{ failure() }}
98-
uses: actions/upload-artifact@v3
99-
with:
100-
name: test_report ${{ matrix.project.PART_NAME }}
101-
path: utbot-framework/build/reports/tests/test/*
89+
- name: Upload UTBot temp directory content
90+
if: ${{ always() }}
91+
uses: actions/upload-artifact@v3
92+
with:
93+
name: utbot_temp ${{ matrix.project.PART_NAME }}
94+
path: |
95+
/tmp/UTBot/generated*/*
96+
/tmp/UTBot/utbot-childprocess-errors/*
97+
- name: Upload test report if tests have failed
98+
if: ${{ failure() }}
99+
uses: actions/upload-artifact@v3
100+
with:
101+
name: test_report ${{ matrix.project.PART_NAME }}
102+
path: utbot-framework-test/build/reports/tests/test/*
102103

103104

104105
project:
105106
needs: prepare-tests-matrix
106-
# Using matrices let create multiple jobs runs based on the combinations of the variables from matrices.
107+
# Using matrices let create multiple jobs runs based on the combinations of the variables from matrices.
107108
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
108109
strategy:
109-
# The option forces to execute all jobs even though some of them have failed.
110+
# The option forces to execute all jobs even though some of them have failed.
110111
fail-fast: false
111112
matrix:
112-
project: [utbot-api, utbot-cli, utbot-core, utbot-framework-api, utbot-fuzzers, utbot-gradle, utbot-instrumentation, utbot-instrumentation-tests, utbot-intellij, utbot-junit-contest, utbot-rd, utbot-sample, utbot-summary, utbot-summary-tests]
113+
project: [utbot-api, utbot-cli, utbot-core, utbot-framework, utbot-framework-api, utbot-fuzzers, utbot-gradle, utbot-instrumentation, utbot-instrumentation-tests, utbot-intellij, utbot-junit-contest, utbot-rd, utbot-sample, utbot-summary, utbot-summary-tests]
113114
runs-on: ubuntu-20.04
114115
container: unittestbot/java-env:java11-zulu-jdk-gradle7.4.2-kotlinc1.7.0
115116
steps:
116-
- name: Print environment variables
117-
run: printenv
117+
- name: Print environment variables
118+
run: printenv
118119

119-
- name: Checkout repository
120-
uses: actions/checkout@v3
120+
- name: Checkout repository
121+
uses: actions/checkout@v3
121122

122-
- name: Check out ${{ github.event.inputs.commit_sha }} commit
123-
if: github.event.inputs.commit_sha != ''
124-
run: |
125-
git fetch
126-
git checkout ${{ github.event.inputs.commit_sha }}
127-
- uses: actions/checkout@v3
128-
with:
129-
ref: ${{ env.COMMIT_SHA }}
123+
- name: Check out ${{ github.event.inputs.commit_sha }} commit
124+
if: github.event.inputs.commit_sha != ''
125+
run: |
126+
git fetch
127+
git checkout ${{ github.event.inputs.commit_sha }}
128+
- uses: actions/checkout@v3
129+
with:
130+
ref: ${{ env.COMMIT_SHA }}
130131

131-
- name: Run monitoring
132-
run: |
133-
echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
134-
chmod +x ./scripts/project/monitoring.sh
135-
./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
136-
- name: Run tests
137-
run: |
138-
cd ${{ matrix.project }}
139-
gradle build --no-daemon
140-
- name: Upload test report if tests have failed
141-
if: ${{ failure() }}
142-
uses: actions/upload-artifact@v3
143-
with:
144-
name: test_report ${{ matrix.project }}
145-
path: ${{ matrix.project }}/build/reports/tests/test/*
132+
- name: Run monitoring
133+
run: |
134+
echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
135+
chmod +x ./scripts/project/monitoring.sh
136+
./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
137+
- name: Run tests
138+
run: |
139+
cd ${{ matrix.project }}
140+
gradle build --no-daemon
141+
- name: Upload test report if tests have failed
142+
if: ${{ failure() }}
143+
uses: actions/upload-artifact@v3
144+
with:
145+
name: test_report ${{ matrix.project }}
146+
path: ${{ matrix.project }}/build/reports/tests/test/*

‎gradle.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ antlr_version=4.9.2
2727
kryo_version=5.3.0
2828
kryo_serializers_version=0.45
2929
asm_version=9.2
30-
testng_version=7.4.0
30+
testng_version=7.6.0
3131
mockito_inline_version=4.0.0
3232
jackson_version = 2.12.3
3333
javasmt_solver_z3_version=4.8.9-sosy1

‎settings.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,6 @@ include 'utbot-summary'
2727
include 'utbot-gradle'
2828
//include 'utbot-maven'
2929
include 'utbot-summary-tests'
30+
include 'utbot-framework-test'
3031
include 'utbot-rd'
3132

‎utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@ import org.junit.jupiter.api.Assertions
55
import org.junit.jupiter.api.BeforeAll
66
import org.junit.jupiter.api.Test
77
import org.utbot.analytics.EngineAnalyticsContext
8-
import org.utbot.examples.UtValueTestCaseChecker
9-
import org.utbot.examples.eq
10-
import org.utbot.examples.withFeaturePath
8+
import org.utbot.tests.infrastructure.UtValueTestCaseChecker
9+
import org.utbot.testcheckers.eq
10+
import org.utbot.testcheckers.withFeaturePath
1111
import java.io.File
1212
import java.io.FileInputStream
1313

14-
class FeatureProcessorWithRepetitionTest : UtValueTestCaseChecker(OnePath::class, false) {
14+
class FeatureProcessorWithRepetitionTest: UtValueTestCaseChecker(OnePath::class, false) {
1515
companion object {
1616
const val featureDir = "src/test/resources/features"
1717
fun reward(coverage: Double, time: Double) = RewardEstimator.reward(coverage, time)

‎utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ package org.utbot.predictors
22

33
import org.junit.jupiter.api.Assertions.assertEquals
44
import org.junit.jupiter.api.Test
5-
import org.utbot.examples.withPathSelectorType
6-
import org.utbot.examples.withRewardModelPath
75
import org.utbot.framework.PathSelectorType
86
import org.utbot.framework.UtSettings
7+
import org.utbot.testcheckers.withPathSelectorType
8+
import org.utbot.testcheckers.withRewardModelPath
99

1010
class LinearStateRewardPredictorTest {
1111
@Test

‎utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ package org.utbot.predictors
33
import org.junit.jupiter.api.Assertions.assertEquals
44
import org.junit.jupiter.api.Disabled
55
import org.junit.jupiter.api.Test
6-
import org.utbot.examples.withPathSelectorType
76
import org.utbot.analytics.StateRewardPredictor
8-
import org.utbot.examples.withRewardModelPath
97
import org.utbot.framework.PathSelectorType
108
import org.utbot.framework.UtSettings
9+
import org.utbot.testcheckers.withPathSelectorType
10+
import org.utbot.testcheckers.withRewardModelPath
1111
import kotlin.system.measureNanoTime
1212

1313
class NNStateRewardPredictorTest {

‎utbot-cli/build.gradle

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
apply from: "${parent.projectDir}/gradle/include/jvm-project.gradle"
22

3+
compileKotlin {
4+
kotlinOptions {
5+
jvmTarget = JavaVersion.VERSION_11
6+
freeCompilerArgs += ["-Xallow-result-return-type", "-Xsam-conversions=class"]
7+
}
8+
}
9+
10+
java {
11+
sourceCompatibility = JavaVersion.VERSION_1_8
12+
targetCompatibility = JavaVersion.VERSION_11
13+
}
14+
15+
316
//noinspection GroovyAssignabilityCheck
417
configurations {
518
fetchInstrumentationJar

‎utbot-framework/src/test/java/org/utbot/examples/StaticInitializerExample.java renamed to ‎utbot-core/src/test/java/org/utbot/examples/StaticInitializerExample.java

File renamed without changes.

‎utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/MatcherUtils.kt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package org.utbot.testcheckers
2+
3+
fun ge(count: Int) = ExecutionsNumberMatcher("ge $count") { it >= count }
4+
5+
fun eq(count: Int) = ExecutionsNumberMatcher("eq $count") { it == count }
6+
7+
fun between(bounds: IntRange) = ExecutionsNumberMatcher("$bounds") { it in bounds }
8+
9+
class ExecutionsNumberMatcher(private val description: String, private val cmp: (Int) -> Boolean) {
10+
operator fun invoke(x: Int) = cmp(x)
11+
override fun toString() = description
12+
}

0 commit comments

Comments
 (0)