Introduce utbot-framework-test project

.github/workflows/build-and-run-tests-from-branch.yml

@@ -30,10 +30,10 @@ jobs:
     steps:
       - name: Print environment variables
         run: printenv
-        
+
       - name: Checkout repository
         uses: actions/checkout@v3
-      
+
       - name: Check out ${{ github.event.inputs.commit_sha }} commit
         if: github.event.inputs.commit_sha != ''
         run: |
@@ -45,101 +45,102 @@ jobs:
           TASKS=$(echo $(cat .github/workflows/framework-tests-matrix.json))
           echo "::set-output name=matrix::$TASKS"
           echo $TASKS
-  framework:
+
+  framework-test:
     # This job does not need to wait for 'prepare-tests-matrix' result.
-    # GitHub allocates runners portionally. Framework tests are time consuming. That's why we want to force them 
+    # GitHub allocates runners portionally. Framework tests are time consuming. That's why we want to force them
     # to start execution early.
     needs: prepare-tests-matrix
-    # Using matrices let create multiple jobs runs based on the combinations of the variables from matrices. 
+    # Using matrices let create multiple jobs runs based on the combinations of the variables from matrices.
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     strategy:
-    # The option forces to execute all jobs even though some of them have failed.
+      # The option forces to execute all jobs even though some of them have failed.
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare-tests-matrix.outputs.matrix) }}
     runs-on: ubuntu-20.04
     container: unittestbot/java-env:java11-zulu-jdk-gradle7.4.2-kotlinc1.7.0
     steps:
-    - name: Print environment variables
-      run: printenv
-
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Print environment variables
+        run: printenv
 
-    - name: Check out ${{ github.event.inputs.commit_sha }} commit
-      if: github.event.inputs.commit_sha != ''
-      run: |
-        git fetch
-        git checkout ${{ github.event.inputs.commit_sha }}
-    - name: Run monitoring
-      run: |
-        echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
-        chmod +x ./scripts/project/monitoring.sh
-        ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
-    - name: Run tests
-      run: |     
-        gradle --no-daemon :utbot-framework:test ${{ matrix.project.TESTS_TO_RUN }}
-    - name: Upload logs
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: logs ${{ matrix.project.PART_NAME }}
-        path: utbot-framework/logs/*
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Check out ${{ github.event.inputs.commit_sha }} commit
+        if: github.event.inputs.commit_sha != ''
+        run: |
+          git fetch
+          git checkout ${{ github.event.inputs.commit_sha }}
+      - name: Run monitoring
+        run: |
+          echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
+          chmod +x ./scripts/project/monitoring.sh
+          ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
+      - name: Run tests
+        run: |
+          gradle --no-daemon :utbot-framework-test:test ${{ matrix.project.TESTS_TO_RUN }}
+      - name: Upload logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs ${{ matrix.project.PART_NAME }}
+          path: utbot-framework-test/logs/*
 
-    - name: Upload UTBot temp directory content
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: utbot_temp ${{ matrix.project.PART_NAME }}
-        path: |
-          /tmp/UTBot/generated*/*
-          /tmp/UTBot/utbot-childprocess-errors/*
-    - name: Upload test report if tests have failed
-      if: ${{ failure() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: test_report ${{ matrix.project.PART_NAME }}
-        path: utbot-framework/build/reports/tests/test/*
+      - name: Upload UTBot temp directory content
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: utbot_temp ${{ matrix.project.PART_NAME }}
+          path: |
+            /tmp/UTBot/generated*/*
+            /tmp/UTBot/utbot-childprocess-errors/*
+      - name: Upload test report if tests have failed
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_report ${{ matrix.project.PART_NAME }}
+          path: utbot-framework-test/build/reports/tests/test/*
 
 
   project:
     needs: prepare-tests-matrix
-    # Using matrices let create multiple jobs runs based on the combinations of the variables from matrices. 
+    # Using matrices let create multiple jobs runs based on the combinations of the variables from matrices.
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     strategy:
-    # The option forces to execute all jobs even though some of them have failed.
+      # The option forces to execute all jobs even though some of them have failed.
       fail-fast: false
       matrix:
-        project: [utbot-api, utbot-cli, utbot-core, utbot-framework-api, utbot-fuzzers, utbot-gradle, utbot-instrumentation, utbot-instrumentation-tests, utbot-intellij, utbot-junit-contest, utbot-rd, utbot-sample, utbot-summary, utbot-summary-tests]
+        project: [utbot-api, utbot-cli, utbot-core, utbot-framework, utbot-framework-api, utbot-fuzzers, utbot-gradle, utbot-instrumentation, utbot-instrumentation-tests, utbot-intellij, utbot-junit-contest, utbot-rd, utbot-sample, utbot-summary, utbot-summary-tests]
     runs-on: ubuntu-20.04
     container: unittestbot/java-env:java11-zulu-jdk-gradle7.4.2-kotlinc1.7.0
     steps:
-    - name: Print environment variables
-      run: printenv
+      - name: Print environment variables
+        run: printenv
 
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-    - name: Check out ${{ github.event.inputs.commit_sha }} commit
-      if: github.event.inputs.commit_sha != ''
-      run: |
-        git fetch
-        git checkout ${{ github.event.inputs.commit_sha }}
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ env.COMMIT_SHA }}
+      - name: Check out ${{ github.event.inputs.commit_sha }} commit
+        if: github.event.inputs.commit_sha != ''
+        run: |
+          git fetch
+          git checkout ${{ github.event.inputs.commit_sha }}
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.COMMIT_SHA }}
 
-    - name: Run monitoring
-      run: |
-        echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
-        chmod +x ./scripts/project/monitoring.sh
-        ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
-    - name: Run tests
-      run: |
-        cd ${{ matrix.project }}
-        gradle build --no-daemon
-    - name: Upload test report if tests have failed
-      if: ${{ failure() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: test_report ${{ matrix.project }}
-        path: ${{ matrix.project }}/build/reports/tests/test/*
+      - name: Run monitoring
+        run: |
+          echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
+          chmod +x ./scripts/project/monitoring.sh
+          ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
+      - name: Run tests
+        run: |
+          cd ${{ matrix.project }}
+          gradle build --no-daemon
+      - name: Upload test report if tests have failed
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_report ${{ matrix.project }}
+          path: ${{ matrix.project }}/build/reports/tests/test/*

gradle.properties

@@ -27,7 +27,7 @@ antlr_version=4.9.2
 kryo_version=5.3.0
 kryo_serializers_version=0.45
 asm_version=9.2
-testng_version=7.4.0
+testng_version=7.6.0
 mockito_inline_version=4.0.0
 jackson_version = 2.12.3
 javasmt_solver_z3_version=4.8.9-sosy1

settings.gradle

@@ -27,5 +27,6 @@ include 'utbot-summary'
 include 'utbot-gradle'
 //include 'utbot-maven'
 include 'utbot-summary-tests'
+include 'utbot-framework-test'
 include 'utbot-rd'
 

utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt

@@ -5,13 +5,13 @@ import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.Test
 import org.utbot.analytics.EngineAnalyticsContext
-import org.utbot.examples.UtValueTestCaseChecker
-import org.utbot.examples.eq
-import org.utbot.examples.withFeaturePath
+import org.utbot.tests.infrastructure.UtValueTestCaseChecker
+import org.utbot.testcheckers.eq
+import org.utbot.testcheckers.withFeaturePath
 import java.io.File
 import java.io.FileInputStream
 
-class FeatureProcessorWithRepetitionTest : UtValueTestCaseChecker(OnePath::class, false) {
+class FeatureProcessorWithRepetitionTest: UtValueTestCaseChecker(OnePath::class, false) {
     companion object {
         const val featureDir = "src/test/resources/features"
         fun reward(coverage: Double, time: Double) = RewardEstimator.reward(coverage, time)

utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt

@@ -2,10 +2,10 @@ package org.utbot.predictors
 
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Test
-import org.utbot.examples.withPathSelectorType
-import org.utbot.examples.withRewardModelPath
 import org.utbot.framework.PathSelectorType
 import org.utbot.framework.UtSettings
+import org.utbot.testcheckers.withPathSelectorType
+import org.utbot.testcheckers.withRewardModelPath
 
 class LinearStateRewardPredictorTest {
     @Test

utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt

@@ -3,11 +3,11 @@ package org.utbot.predictors
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
-import org.utbot.examples.withPathSelectorType
 import org.utbot.analytics.StateRewardPredictor
-import org.utbot.examples.withRewardModelPath
 import org.utbot.framework.PathSelectorType
 import org.utbot.framework.UtSettings
+import org.utbot.testcheckers.withPathSelectorType
+import org.utbot.testcheckers.withRewardModelPath
 import kotlin.system.measureNanoTime
 
 class NNStateRewardPredictorTest {

utbot-cli/build.gradle

@@ -1,5 +1,18 @@
 apply from: "${parent.projectDir}/gradle/include/jvm-project.gradle"
 
+compileKotlin {
+    kotlinOptions {
+        jvmTarget = JavaVersion.VERSION_11
+        freeCompilerArgs += ["-Xallow-result-return-type", "-Xsam-conversions=class"]
+    }
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_1_8
+    targetCompatibility = JavaVersion.VERSION_11
+}
+
+
 //noinspection GroovyAssignabilityCheck
 configurations {
     fetchInstrumentationJar

utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/MatcherUtils.kt

@@ -0,0 +1,12 @@
+package org.utbot.testcheckers
+
+fun ge(count: Int) = ExecutionsNumberMatcher("ge $count") { it >= count }
+
+fun eq(count: Int) = ExecutionsNumberMatcher("eq $count") { it == count }
+
+fun between(bounds: IntRange) = ExecutionsNumberMatcher("$bounds") { it in bounds }
+
+class ExecutionsNumberMatcher(private val description: String, private val cmp: (Int) -> Boolean) {
+    operator fun invoke(x: Int) = cmp(x)
+    override fun toString() = description
+}