Do not run acceptance tests for PR that does not change SBO images or testing framework (#1150)

pmacik · web-flow · commit ad9085f4827d · 2022-05-25T13:58:14.000-04:00
Signed-off-by: Pavel Macík &lt;pavel.macik@gmail.com&gt;
diff --git a/.github/actions/check-skip-acceptance-tests/action.yaml b/.github/actions/check-skip-acceptance-tests/action.yaml
@@ -0,0 +1,77 @@
+# action.yml
+name: 'Check if acceptance tests can be skipped'
+description: 'Check if acceptance tests can be skipped based on the PR content'
+outputs:
+  can_skip:
+    description: "true if acceptance teststests can be skipped"
+    value: ${{ steps.check-skip-test.outputs.result }}
+runs:
+  using: "composite"
+  steps:
+    - id: check-skip-test
+      uses: actions/github-script@v6.1.0
+      with:
+        result-encoding: string
+        script: |
+          let currentCount = 0;
+          let currentPage = 1;
+          let overallCount = 0;
+          let pageSize = 100;
+          let maxPages = 30;
+
+          do {
+            const result = await github.rest.pulls.listFiles({
+              owner: context.payload.repository.owner.login,
+              repo: context.payload.repository.name,
+              pull_number: context.payload.number,
+              per_page: pageSize,
+              page: currentPage
+            })
+
+            let fileSet = result.data.filter(f =>
+                f.filename.startsWith(".github/actions/collect-kube-resources") ||
+                f.filename.startsWith(".github/actions/setup-cli") ||
+                f.filename.startsWith(".github/actions/setup-podman") ||
+                f.filename.startsWith(".github/workflows/pr-checks") ||
+                f.filename.startsWith("apis") ||
+                f.filename.startsWith("build") ||
+                f.filename.startsWith("config") ||
+                f.filename.startsWith("controllers") ||
+                f.filename.startsWith("hack/get-test-namespace") ||
+                f.filename.startsWith("hack/remove-sbr-finalizers.sh") ||
+                f.filename.startsWith("hack/test-cleanup.sh") ||
+                f.filename.startsWith("controllers") ||
+                f.filename.startsWith("make/acceptance.mk") ||
+                f.filename.startsWith("make/build.mk") ||
+                f.filename.startsWith("make/common.mk") ||
+                f.filename.startsWith("make/release.mk") ||
+                f.filename.startsWith("make/version.mk") ||
+                f.filename.startsWith("pkg") ||
+                f.filename.startsWith("test/acceptance") ||
+                f.filename.startsWith("tools") ||
+                f.filename.startsWith("vendor") ||
+                f.filename.startsWith("go.mod") ||
+                f.filename.startsWith("go.sum") ||
+                f.filename.startsWith("install.sh") ||
+                f.filename.startsWith("main.go")
+              )
+            fileSet.forEach(i => console.log(" > " + i.status + ": " + i.filename))
+            overallCount = overallCount + fileSet.length
+            currentCount = result.data.length
+            if (currentCount == pageSize){
+              currentPage = currentPage + 1
+            }
+          } while (currentCount == pageSize && currentPage <= maxPages)
+
+          const canSkip = (overallCount == 0 && currentPage <= maxPages)
+          if(canSkip){
+            console.log("The PR changes neither SBO release, images nor manifests, nor the acceptance testing framework, nor the related CI. Execution of the acceptance tests CAN be skipped. (See https://issues.redhat.com/browse/APPSVC-1116)")
+          } else {
+            if (currentPage > maxPages) {
+              console.log("Overall the PR changes more than the maximum number of " + (pageSize * maxPages) + " files and so cannot be determined if any of those beyond effect SBO release, images or manifests, or the acceptance testing framework, or the related CI. The acceptance tests CAN NOT be skipped.")
+            } else {
+              console.log("The PR changes files that effect SBO release, images or manifests, or the acceptance testing framework, or the related CI. The acceptance tests CAN NOT be skipped.")
+            }
+          }
+
+          return canSkip
diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml
@@ -74,18 +74,25 @@ jobs:
       - name: Checkout Git Repository
         uses: actions/checkout@v2
 
+      - name: Check if acceptance tests can be skipped
+        id: check-skip-acceptance
+        uses: ./.github/actions/check-skip-acceptance-tests
+
       - name: Set up Python
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: actions/setup-python@v2
         with:
           python-version: "3.7"
           architecture: "x64"
 
       - name: Setup-cli
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: ./.github/actions/setup-cli
         with:
           start-minikube: true
 
       - name: Wait for push
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: lewagon/wait-on-check-action@1b1630e169116b58a4b933d5ad7effc46d3d312d
         with:
           ref: ${{ github.event.pull_request.head.sha }}
@@ -94,11 +101,13 @@ jobs:
           wait-interval: 60
 
       - name: Extract image references
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: marcofaggian/action-download-multiple-artifacts@v3.0.8
         with:
           names: operator-refs-${{github.event.pull_request.number}}-${{github.event.pull_request.head.sha}}
 
       - name: Acceptance tests
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         timeout-minutes: 60
         run: |
           source ./operator.refs
@@ -107,31 +116,31 @@ jobs:
           make SKIP_REGISTRY_LOGIN=true -o registry-login test-acceptance-with-bundle
 
       - name: Collect Kube resources
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         continue-on-error: true
         uses: ./.github/actions/collect-kube-resources
         with:
           operator-namespace: operators
           olm-namespace: olm
           test-namespace-file: out/test-namespace
           output-path: ${{env.TEST_RESULTS}}
-        if: always()
 
       - name: Setup Testspace
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{ github.repository_owner }}
-        if: always()
 
       - name: Publish tests results to Testspace
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         run: |
           testspace [${{ env.TEST_RUN }}]${{ env.TEST_RESULTS }}/TEST*.xml
-        if: always()
 
       - uses: actions/upload-artifact@v2
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         with:
           name: kubernetes-with-olm-test-results
           path: ${{ env.TEST_RESULTS }}
-        if: always()
 
   acceptance-supported-operators:
     name: Supported Operators Acceptance Tests with Kubernetes and using OLM
@@ -145,18 +154,25 @@ jobs:
       - name: Checkout Git Repository
         uses: actions/checkout@v2
 
+      - name: Check if acceptance tests can be skipped
+        id: check-skip-acceptance
+        uses: ./.github/actions/check-skip-acceptance-tests
+
       - name: Set up Python
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: actions/setup-python@v2
         with:
           python-version: "3.7"
           architecture: "x64"
 
       - name: Setup-cli
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: ./.github/actions/setup-cli
         with:
           start-minikube: true
 
       - name: Wait for push
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: lewagon/wait-on-check-action@1b1630e169116b58a4b933d5ad7effc46d3d312d
         with:
           ref: ${{ github.event.pull_request.head.sha }}
@@ -165,11 +181,13 @@ jobs:
           wait-interval: 60
 
       - name: Extract image references
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: marcofaggian/action-download-multiple-artifacts@v3.0.8
         with:
           names: operator-refs-${{github.event.pull_request.number}}-${{github.event.pull_request.head.sha}}
 
       - name: Acceptance tests
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         timeout-minutes: 60
         run: |
           source ./operator.refs
@@ -178,31 +196,31 @@ jobs:
           make SKIP_REGISTRY_LOGIN=true -o registry-login test-acceptance-with-bundle
 
       - name: Collect Kube resources
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         continue-on-error: true
         uses: ./.github/actions/collect-kube-resources
         with:
           operator-namespace: operators
           olm-namespace: olm
           test-namespace-file: out/test-namespace
           output-path: ${{env.TEST_RESULTS}}
-        if: always()
 
       - name: Setup Testspace
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{ github.repository_owner }}
-        if: always()
 
       - name: Publish tests results to Testspace
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         run: |
           testspace [${{ env.TEST_RUN }}]${{ env.TEST_RESULTS }}/TEST*.xml
-        if: always()
 
       - uses: actions/upload-artifact@v2
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         with:
           name: supported-operators-kubernetes
           path: ${{ env.TEST_RESULTS }}
-        if: always()
 
   acceptance_without_olm:
     name: Acceptance tests running on Kubernetes without using OLM
@@ -217,29 +235,38 @@ jobs:
       - name: Checkout Git Repository
         uses: actions/checkout@v2
 
+      - name: Check if acceptance tests can be skipped
+        id: check-skip-acceptance
+        uses: ./.github/actions/check-skip-acceptance-tests
+
       - name: Set up Python
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: actions/setup-python@v2
         with:
           python-version: "3.7"
           architecture: "x64"
 
       - name: Set up CLI
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: ./.github/actions/setup-cli
         with:
           start-minikube: true
 
       - name: Set up Go
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: actions/setup-go@v2
         with:
           go-version: "^1.16"
 
       - name: Setup umoci cli
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         run: |
           curl -Lo umoci https://github.com/opencontainers/umoci/releases/download/v${UMOCI_VERSION}/umoci.amd64
           chmod +x umoci
           mv -v umoci $GITHUB_WORKSPACE/bin/
 
       - name: Wait for push
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: lewagon/wait-on-check-action@1b1630e169116b58a4b933d5ad7effc46d3d312d
         with:
           ref: ${{ github.event.pull_request.head.sha }}
@@ -248,11 +275,13 @@ jobs:
           wait-interval: 60
 
       - name: Extract image references
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: marcofaggian/action-download-multiple-artifacts@v3.0.8
         with:
           names: operator-refs-${{github.event.pull_request.number}}-${{github.event.pull_request.head.sha}}
 
       - name: Acceptance tests against vanilla k8s without OLM
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         timeout-minutes: 60
         run: |
           source ./operator.refs
@@ -264,30 +293,30 @@ jobs:
           make TEST_ACCEPTANCE_START_SBO=remote test-acceptance
 
       - name: Collect Kube resources
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         continue-on-error: true
         uses: ./.github/actions/collect-kube-resources
         with:
           operator-namespace: service-binding-operator
           test-namespace-file: out/test-namespace
           output-path: ${{env.TEST_RESULTS}}
-        if: always()
 
       - name: Setup Testspace
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{ github.repository_owner }}
-        if: always()
 
       - name: Publish tests results to Testspace
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         run: |
           testspace [${{ env.TEST_RUN }}]${{ env.TEST_RESULTS }}/TEST*.xml
-        if: always()
 
       - uses: actions/upload-artifact@v2
+        if: ${{ steps.check-skip-acceptance.outputs.can_skip != 'true' }}
         with:
           name: kubernetes-without-olm-test-results
           path: ${{ env.TEST_RESULTS }}
-        if: always()
 
   single-commit:
     name: Single commit PR
