Skip to content

simplify smoke-test action #5311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Mar 27, 2024
Merged

simplify smoke-test action #5311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5da4138
simplify smoke-test action
pdabelf5 Mar 26, 2024
2161686
always build the image for now
pdabelf5 Mar 26, 2024
96dfd7a
correct secrets for image build
pdabelf5 Mar 26, 2024
31bd170
correct secrets for image build
pdabelf5 Mar 26, 2024
2676ff5
add registry option to makefile
pdabelf5 Mar 26, 2024
f435d90
add image-type to action
pdabelf5 Mar 26, 2024
38bd1e8
fix
pdabelf5 Mar 26, 2024
022f00d
ensure docker is setup to build
pdabelf5 Mar 26, 2024
da364eb
Merge branch 'main' into smoke-action
pdabelf5 Mar 26, 2024
1c4591b
update to use image-type for ic-type
pdabelf5 Mar 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 13 additions & 98 deletions .github/actions/smoke-tests/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,51 +2,32 @@ name: Run Smoke Tests
description: Run Smoke Tests for the project

inputs:
go-md5:
required: true
type: string
base-image-md5:
required: true
type: string
k8s-version:
description: Kubernetes version to use
required: false
k8s-timeout:
description: Timeout to use
default: 75s
required: false
image:
description: Docker image to use
default: debian
required: false
image-type:
description: Image type to test
required: true
image-name:
description: Docker image name to test
required: true
tag:
description: Docker image tag to test
required: true
test-image:
description: Test Docker image to use
default: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:latest
required: false
marker:
description: Marker to use
required: false
nginx-key:
description: Nginx key to use
required: false
nginx-crt:
description: Nginx cert to use
required: false
azure-ad-secret:
description: Azure Active Directory secret for JWKs
required: false
rhel-license:
description: RHEL license for UBI builds
required: false
gcr-workload-identity-secret:
description: Google Workflow Identity secret
required: false
gcr-service-account-secret:
description: Google Service Account secret
required: false
forked-workflow:
description: Does this workflow full access to repo resources?
required: false

outputs:
test-results-name:
Expand All @@ -56,90 +37,24 @@ outputs:
runs:
using: composite
steps:
- name: Fetch Cached Artifacts
uses: actions/cache@v3
with:
path: ${{ github.workspace }}/dist
key: nginx-ingress-${{ inputs.go-md5 }}

- name: Ingress type
id: ingress-type
run: |
echo "name=nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
echo "tag=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}" >> $GITHUB_OUTPUT
echo "modules=${{ contains(inputs.marker, 'appprotect') && 'waf' || '' }}${{ contains(inputs.marker, 'dos') && 'dos' || '' }}" >> $GITHUB_OUTPUT
shell: bash

- name: Docker Buildx
uses: docker/setup-buildx-action@v2

- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v2
with:
token_format: access_token
workload_identity_provider: ${{ inputs.gcr-workload-identity-secret }}
service_account: ${{ inputs.gcr-service-account-secret }}
if: ${{ inputs.forked-workflow == 'false' }}

- name: Login to GCR
uses: docker/login-action@v3
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}
if: ${{ inputs.forked-workflow == 'false' }}

- name: Build ${{ inputs.image }} Container
uses: docker/build-push-action@v3
with:
file: build/Dockerfile
context: "."
cache-from: type=gha,scope=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }}
target: goreleaser${{ inputs.forked-workflow == 'false' && '-prebuilt' || '' }}
tags: "docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }}"
load: true
pull: true
build-args: |
BUILD_OS=${{ inputs.image }}
PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ contains(inputs.image, 'plus') && 'plus' || 'oss' }}:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ contains(inputs.marker, 'appprotect') && '-waf' || '' }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}
IC_VERSION=CI
${{ steps.ingress-type.outputs.modules != '' && format('NAP_MODULES={0}', steps.ingress-type.outputs.modules) || '' }}
${{ contains(inputs.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }}
secrets: |
${{ contains(inputs.image, 'plus') && format('"nginx-repo.crt={0}"', inputs.nginx-crt) || '' }}
${{ contains(inputs.image, 'plus') && format('"nginx-repo.key={0}"', inputs.nginx-key) || '' }}
${{ contains(inputs.image, 'ubi') && format('"rhel_license={0}"', inputs.rhel-license) || '' }}

- name: Deploy Kubernetes
id: k8s
run: |
make -f tests/Makefile create-kind-cluster K8S_CLUSTER_NAME=${{ github.run_id }} K8S_CLUSTER_VERSION=${{ inputs.k8s-version }} K8S_TIMEOUT=${{ inputs.k8s-timeout }}
make -f tests/Makefile image-load PREFIX=nginx/${{ steps.ingress-type.outputs.name }} TAG=${{ steps.ingress-type.outputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }}
make -f tests/Makefile image-load REGISTRY="" PREFIX=${{ inputs.image-name }} TAG=${{ inputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }}
marker="${{ inputs.marker }}"
nospaces="${marker// /_}"
sanitized_marker="${nospaces//\'/}"
name="${sanitized_marker:-${{ inputs.k8s-version }}}"
echo "cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)" >> $GITHUB_OUTPUT
echo "cluster=$(echo nginx-${{ inputs.image }}-$name)" >> $GITHUB_OUTPUT
echo "cluster=$(echo nginx-${{ inputs.image-type }}-$name)" >> $GITHUB_OUTPUT
shell: bash

- name: Setup Kubeconfig
run: |
sed -i 's|server:.*|server: https://${{ steps.k8s.outputs.cluster_ip }}:6443|' ~/.kube/kind/config
shell: bash

- name: Build Test-Runner Container
uses: docker/build-push-action@v3
with:
file: tests/Dockerfile
context: "."
cache-from: type=gha,scope=test-runner
tags: ${{ inputs.test-image }}
pull: true
load: true
if: ${{ inputs.forked-workflow == 'true' }}

- name: Run Smoke Tests
run: |
touch tests-${{ steps.k8s.outputs.cluster }}.html
Expand All @@ -149,9 +64,9 @@ runs:
-v ${{ github.workspace }}/tests:/workspace/tests \
-v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \
--context=kind-${{ github.run_id }} \
--image=docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }} \
--image=${{ inputs.image-name }}:${{ inputs.tag }} \
--image-pull-policy=Never \
--ic-type=${{ steps.ingress-type.outputs.name }} \
--ic-type=nginx${{ contains(inputs.image-type, 'plus') && '-plus' || '' }}-ingress \
--service=nodeport --node-ip=${{ steps.k8s.outputs.cluster_ip }} \
--html=tests-${{ steps.k8s.outputs.cluster }}.html \
--self-contained-html \
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/build-plus.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ jobs:
build-args: |
BUILD_OS=${{ inputs.image }}
IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }}
${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
secrets: |
"nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}"
"nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}"
Expand Down Expand Up @@ -204,8 +204,8 @@ jobs:
BUILD_OS=${{ inputs.image }}
PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/plus:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }}
IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }}
${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', inputs.nap_modules) || '' }}
${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
${{ (contains(inputs.target, 'aws') && inputs.nap_modules != '') && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
secrets: |
"nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}"
"nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}"
Expand Down
82 changes: 62 additions & 20 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,11 +436,11 @@ jobs:
tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
pull: true
push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
load: ${{ needs.checks.outputs.forked_workflow == 'true' }}
if: ${{ steps.check-image.outcome == 'failure' || needs.checks.outputs.forked_workflow == 'true' }}
load: false
if: ${{ steps.check-image.outcome == 'failure' }}

smoke-tests:
name: ${{ matrix.images.label }} ${{ matrix.images.image }} smoke tests
name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
runs-on: ubuntu-22.04
needs: [checks, setup-matrix]
strategy:
Expand All @@ -456,8 +456,10 @@ jobs:
- name: NAP modules
id: nap_modules
run: |
[[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || modules="${{ matrix.images.nap_modules }}"
[[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || name="${{ matrix.images.nap_modules }}"
[[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="both" || modules="${{ matrix.images.nap_modules }}"
echo "modules=${modules}" >> $GITHUB_OUTPUT
echo "name=${name}" >> $GITHUB_OUTPUT
if: ${{ matrix.images.nap_modules }}

- name: Authenticate to Google Cloud
Expand All @@ -480,7 +482,7 @@ jobs:
- name: Check if base images exist
id: base_exists
run: |
if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}; then
if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}; then
echo "exists=0" >> $GITHUB_OUTPUT
else
echo "exists=1" >> $GITHUB_OUTPUT
Expand All @@ -489,53 +491,93 @@ jobs:

- name: Docker Buildx
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }}

- name: Setup QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
with:
platforms: ${{ matrix.platforms }}
if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }}

- name: Build Base Container
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: build/Dockerfile
context: "."
cache-from: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}
cache-to: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }},mode=max
cache-from: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}
cache-to: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }},mode=max
target: common
tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}
tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}
platforms: ${{ matrix.images.platforms }}
pull: true
push: true
build-args: |
BUILD_OS=${{ matrix.images.image }}
IC_VERSION=${{ needs.checks.outputs.ic_version }}
${{ contains(matrix.images.image, 'nap') && format('NAP_MODULES={0}', matrix.images.nap_modules) || '' }}
${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
secrets: |
${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }}
${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }}
${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}
if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }}

- name: Fetch Cached Artifacts
uses: actions/cache@v3
with:
path: ${{ github.workspace }}/dist
key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}

- name: Check if test image exists
id: check-image
run: |
docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
shell: bash
continue-on-error: true
if: ${{ needs.checks.outputs.forked_workflow == 'false' }}

- name: Build Test-Runner Container
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: tests/Dockerfile
context: "."
cache-from: type=gha,scope=test-runner
tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
pull: true
push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
load: ${{ needs.checks.outputs.forked_workflow == 'true' }}
if: ${{ needs.checks.outputs.forked_workflow == 'true' || steps.check-image.outcome == 'failure' }}

- name: Build ${{ matrix.images.image }} Container
uses: docker/build-push-action@v3
with:
file: build/Dockerfile
context: "."
cache-from: type=gha,scope=${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }}
cache-to: type=gha,scope=${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }},mode=max
target: goreleaser${{ needs.checks.outputs.forked_workflow == 'false' && '-prebuilt' || '' }}
tags: "docker.io/nginx/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress:${{ matrix.images.image }}${{ contains(matrix.images.marker, 'dos') && '-dos' || '' }}${{ contains(matrix.images.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}"
load: true
pull: true
build-args: |
BUILD_OS=${{ matrix.images.image }}
IC_VERSION=CI
PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }}
${{ contains(matrix.images.image, 'nap') && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
${{ contains(matrix.images.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }}
secrets: |
${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }}
${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }}
${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}

- name: Run Smoke Tests
id: smoke-tests
uses: ./.github/actions/smoke-tests
with:
image: ${{ matrix.images.image != '' && matrix.images.image || 'debian' }}
image-type: ${{ matrix.images.image }}
image-name: docker.io/nginx/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress
tag: ${{ matrix.images.image }}${{ contains(matrix.images.marker, 'dos') && '-dos' || '' }}${{ contains(matrix.images.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}
marker: ${{ matrix.images.marker != '' && matrix.images.marker || '' }}
k8s-version: ${{ matrix.k8s }}
nginx-crt: ${{ contains(matrix.images.image, 'nap') && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}
nginx-key: ${{ contains(matrix.images.image, 'nap') && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}
azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
gcr-workload-identity-secret: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
gcr-service-account-secret: ${{ secrets.GCR_SERVICE_ACCOUNT }}
rhel-license: ${{ contains(matrix.images.image, 'ubi') && secrets.RHEL_LICENSE || '' }}
go-md5: ${{ needs.checks.outputs.go_code_md5 }}
base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
forked-workflow: ${{ needs.checks.outputs.forked_workflow == 'true' }}

- name: Upload Test Results
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
Expand Down
Loading