From 19a40104f88dc057005dd126297e2e2a711db133 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 8 Jul 2025 09:12:44 -0700 Subject: [PATCH 1/5] Move to more secure split trivy workflow based on labels, not comments (#12592) Signed-off-by: Derek Nola --- .github/workflows/trivy-scan.yml | 129 ++++++++++++++++++++++++++++ .github/workflows/trivy-trigger.yml | 68 +++++++++++++++ .github/workflows/trivy.yaml | 48 ----------- 3 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/trivy-scan.yml create mode 100644 .github/workflows/trivy-trigger.yml delete mode 100644 .github/workflows/trivy.yaml diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml new file mode 100644 index 000000000000..7af4d7f4b0c2 --- /dev/null +++ b/.github/workflows/trivy-scan.yml @@ -0,0 +1,129 @@ +name: Trivy Scan Result + +on: + workflow_run: + workflows: ["Trivy Scan Trigger"] + types: + - completed + +permissions: + contents: read + +jobs: + trivy_scan: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + permissions: + contents: write # Required to checkout the PR's head SHA. + outputs: + pr_number: ${{ steps.pr_context.outputs.pr_number }} + + steps: + # For some reason with workflow_run.id, download-artifact does not work. + # Github Docs explicity provide an example of using github-script to download artifacts. + - name: 'Download artifact' + uses: actions/github-script@v7 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr-context-for-scan" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + const fs = require('fs'); + fs.writeFileSync('pr-context-for-scan.zip', Buffer.from(download.data)); + + - name: 'Unzip artifact to pr-context' + run: unzip pr-context-for-scan.zip -d pr-context + + - name: Setup PR context + id: pr_context + run: | + pr_number=$(cat pr-context/pr_number) + echo "pr_number=$pr_number" >> $GITHUB_OUTPUT + + - name: Load K3s Image + run: docker load -i pr-context/k3s.tar + + - name: Download Rancher's VEX Hub report + run: curl -fsSO https://raw.githubusercontent.com/rancher/vexhub/refs/heads/main/reports/rancher.openvex.json + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.31.0 + with: + image-ref: 'rancher/k3s:latest' + format: 'table' + severity: "HIGH,CRITICAL" + output: "trivy-report.txt" + env: + TRIVY_VEX: rancher.openvex.json + TRIVY_SHOW_SUPPRESSED: true + + - name: Upload Trivy Report + uses: actions/upload-artifact@v4 + with: + name: trivy-report + path: trivy-report.txt + retention-days: 2 + if-no-files-found: error + + report_results: + needs: trivy_scan + if: always() # Run even if the scan fails. + runs-on: ubuntu-latest + permissions: + pull-requests: write # Required to post comments. + + steps: + - name: Download Trivy Report artifact + uses: actions/download-artifact@v4 + if: needs.trivy_scan.result == 'success' + with: + name: trivy-report + path: . + + - name: Add Trivy Report to PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + SCAN_RESULT: ${{ needs.trivy_scan.result }} + PR_NUMBER: ${{ needs.trivy_scan.outputs.pr_number }} + run: | + if [[ "$SCAN_RESULT" == "failure" ]]; then + gh issue comment $PR_NUMBER -b ":x: Trivy scan action failed, check logs :x:" + exit 0 + fi + + if [ -s trivy-report.txt ] && [ -n "$(grep -v '^\s*$' trivy-report.txt)" ]; then + echo '```' | cat - trivy-report.txt > temp && mv temp trivy-report.txt + echo '```' >> trivy-report.txt + gh issue comment $PR_NUMBER -F trivy-report.txt + else + echo ':star2: No High or Critical CVEs Found :star2:' > trivy-report.txt + gh issue comment $PR_NUMBER -F trivy-report.txt + fi + + remove_label: + if: always() # Run even if the scan fails. + needs: trivy_scan + runs-on: ubuntu-latest + permissions: + pull-requests: write # Required to remove labels from the PR. + + steps: + - name: Remove 'scan-with-trivy' label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ needs.trivy_scan.outputs.pr_number }} + run: | + gh pr edit $PR_NUMBER --remove-label "scan-with-trivy" diff --git a/.github/workflows/trivy-trigger.yml b/.github/workflows/trivy-trigger.yml new file mode 100644 index 000000000000..0628bba64611 --- /dev/null +++ b/.github/workflows/trivy-trigger.yml @@ -0,0 +1,68 @@ +name: Trivy Scan Trigger + +# This workflow is triggered when a pull request is labeled with 'scan-with-trivy'. +# This can only be initiated by a user who is a member of the k3s-io organization and has write permissions. +# It isolates the built of k3s within a unprivileged enviroment. +# The follow up unprivileged workflow will then use the artifact created here to run the scan +# and report the results back to the PR. + +on: + pull_request: + types: [labeled] + +permissions: + contents: read + +jobs: + trigger-scan: + if: github.event.label.name == 'scan-with-trivy' + runs-on: ubuntu-latest + steps: + - name: Verify actor is a member of k3s-io organization and has write permissions + uses: actions/github-script@v7 + with: + script: | + const org = 'k3s-io'; + const actor = context.actor; + const { repo, owner } = context.repo; + + try { + const result = await github.rest.orgs.checkMembershipForUser({ + org, + username: actor, + }); + } catch (error) { + core.setFailed(`User ${actor} is not an public member of the ${org} organization`); + } + + const { data: { permission } } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username: actor + }); + + if (permission !== 'admin' && permission !== 'write') { + core.setFailed(`User @${actor} does not have write permission. Scan can only be triggered by repository collaborators with write access.`); + } + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build And Save K3s Image + run: | + make local-image + make tag-image-latest + docker save -o k3s.tar rancher/k3s:latest + + - name: Create PR context artifact + run: | + mkdir -p pr-context + echo "${{ github.event.pull_request.number }}" > pr-context/pr_number + mv k3s.tar pr-context/k3s.tar + + - name: Upload PR context artifact + uses: actions/upload-artifact@v4 + with: + name: pr-context-for-scan + path: pr-context/ + retention-days: 1 diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml deleted file mode 100644 index ffaa501cb31a..000000000000 --- a/.github/workflows/trivy.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: PR Comment Triggered Trivy Scan - -on: - issue_comment: - types: [created] - -jobs: - trivy_scan: - if: github.event.issue.pull_request && github.event.comment.body == '/trivy' - runs-on: ubuntu-latest - permissions: - pull-requests: write - env: - GH_TOKEN: ${{ github.token }} - steps: - - name: Checkout PR code - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ github.event.issue.number }}/head - - - name: Comment Status on PR - run: | - gh repo set-default ${{ github.repository }} - gh pr comment ${{ github.event.issue.number }} -b ":construction: Running Trivy scan on PR :construction: " - - - name: Build K3s Image - run: | - make local-image - make tag-image-latest - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.24.0 - with: - image-ref: 'rancher/k3s:latest' - format: 'table' - severity: "HIGH,CRITICAL" - output: "trivy-report.txt" - - - name: Add Trivy Report to PR - run: | - echo '```' | cat - trivy-report.txt > temp && mv temp trivy-report.txt - echo '```' >> trivy-report.txt - gh issue comment ${{ github.event.issue.number }} --edit-last -F trivy-report.txt - - - name: Report Failure - if: ${{ failure() }} - run: | - gh issue comment ${{ github.event.issue.number }} --edit-last -b ":x: Trivy scan action failed, check logs :x:" From 03cd10b97a0da76ece96ebb89ec0cc7b7341b314 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 2 Jul 2025 10:24:10 -0700 Subject: [PATCH 2/5] Add basic fuzz test Signed-off-by: Derek Nola --- pkg/authenticator/hash/scrypt_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/authenticator/hash/scrypt_test.go b/pkg/authenticator/hash/scrypt_test.go index cfec9b11a2a9..6937cebda30d 100644 --- a/pkg/authenticator/hash/scrypt_test.go +++ b/pkg/authenticator/hash/scrypt_test.go @@ -37,3 +37,21 @@ func Test_UnitSCrypt_VerifyHash(t *testing.T) { }) } } + +func FuzzVerifyHash(f *testing.F) { + hasher := NewSCrypt() + validSecret := "my-secret-password" + validHash, _ := hasher.CreateHash(validSecret) + + // Seed the fuzzer with some valid and invalid inputs + f.Add(validHash, validSecret) + f.Add(validHash, "wrong-password") + f.Add("", "") + f.Add("$1:deadbeef:f:8:1:corrupt-hash", "any-password") + f.Add("", validSecret) + f.Add(validHash, "") + + f.Fuzz(func(t *testing.T, hash, secretKey string) { + _ = hasher.VerifyHash(hash, secretKey) + }) +} From 04d049e9eaa58fa4269250b007a008f734f67102 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 11 Jul 2025 09:31:47 -0700 Subject: [PATCH 3/5] Add retry around common timeout for hardened docker test (#12601) Signed-off-by: Derek Nola --- tests/docker/hardened/hardened_test.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/docker/hardened/hardened_test.go b/tests/docker/hardened/hardened_test.go index fd10b914bbfc..61dbdb0c39a0 100644 --- a/tests/docker/hardened/hardened_test.go +++ b/tests/docker/hardened/hardened_test.go @@ -85,17 +85,20 @@ kubelet-arg: return docker.RunCommand(cmd) }, "60s", "5s").Should(Equal("2")) _, err = config.DeployWorkload("hardened-netpool.yaml") + Expect(err).NotTo(HaveOccurred()) }) It("checks ingress connections", func() { for _, scheme := range []string{"http", "https"} { - for _, server := range config.Servers { - cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, server.IP) - Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) - } - for _, agent := range config.Agents { - cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, agent.IP) - Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) - } + Eventually(func(g Gomega) { + for _, server := range config.Servers { + cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, server.IP) + g.Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) + } + for _, agent := range config.Agents { + cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, agent.IP) + g.Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) + } + }, "30s", "10s").Should(Succeed()) } }) It("confirms we can make a request through the nodeport service", func() { From 27501fc5c136f6195df3f61266a6aa2a1e2a32e2 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 11 Jul 2025 09:35:13 -0700 Subject: [PATCH 4/5] Remove ghcr build cache (#12602) Signed-off-by: Derek Nola --- .github/workflows/build-k3s.yaml | 25 ++----------------------- .github/workflows/e2e.yaml | 2 -- .github/workflows/integration.yaml | 4 ---- .github/workflows/release.yml | 3 --- 4 files changed, 2 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build-k3s.yaml b/.github/workflows/build-k3s.yaml index 43fc8b280b96..3cc33acb978b 100644 --- a/.github/workflows/build-k3s.yaml +++ b/.github/workflows/build-k3s.yaml @@ -19,20 +19,9 @@ on: description: 'Upload contents of build/out, used to build the k3s image externally' required: false default: false - cache: - type: string - description: 'Cache mode: "read", "write", or empty for no cache' - required: false - default: '' - -# Note that is workflow requires the following permissions: -# contents: read -# If using the cache: write option, you will need: -# packages: write -# If using the cache: read option, you will need: -# packages: read - +permissions: + contents: read jobs: build: @@ -67,14 +56,6 @@ jobs: echo "dirty=${DIRTY}" } >> "$GITHUB_OUTPUT" - - name: Login to GitHub Container Registry - if: inputs.cache == 'write' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build K3s Binary Native if: inputs.arch == 'arm64' || inputs.arch == 'amd64' env: @@ -91,8 +72,6 @@ jobs: TREE_STATE=${{ steps.git_vars.outputs.tree_state }} COMMIT=${{ steps.git_vars.outputs.commit }} DIRTY=${{ steps.git_vars.outputs.dirty }} - cache-from: ${{ inputs.cache != '' && format('type=registry,ref=ghcr.io/{0}:cache-{1}', github.repository, inputs.arch) || '' }} - cache-to: ${{ inputs.cache == 'write' && format('type=registry,ref=ghcr.io/{0}:cache-{1},mode=max', github.repository, inputs.arch) || '' }} push: false provenance: mode=min outputs: type=local,dest=. diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9058bcda238e..89f6fa7ca679 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -35,7 +35,6 @@ jobs: uses: ./.github/workflows/build-k3s.yaml with: upload-image: true - cache: ${{ github.ref == 'refs/heads/master' && 'write' || 'read' }} build-arm64: uses: ./.github/workflows/build-k3s.yaml permissions: @@ -44,7 +43,6 @@ jobs: with: arch: arm64 upload-image: true - cache: ${{ github.ref == 'refs/heads/master' && 'write' || 'read' }} e2e: name: "E2E Tests" needs: build diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 285eeea9716f..0aced62ce112 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -29,13 +29,9 @@ env: jobs: build: - permissions: - contents: read - packages: read uses: ./.github/workflows/build-k3s.yaml with: os: linux - cache: read build-windows: uses: ./.github/workflows/build-k3s.yaml with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 876d3d2c9163..7ae7ee913abc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,6 @@ jobs: name: Build Binary (amd64) uses: ./.github/workflows/build-k3s.yaml with: - cache: '' # No cache for release builds upload-build: true build-arm64: @@ -21,7 +20,6 @@ jobs: uses: ./.github/workflows/build-k3s.yaml with: arch: arm64 - cache: '' # No cache for release builds upload-build: true build-arm: @@ -29,7 +27,6 @@ jobs: uses: ./.github/workflows/build-k3s.yaml with: arch: arm - cache: '' # No cache for release builds upload-build: true push-release-image: From d4f69f708adc6347cd014e87ed84a979693e2f43 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 11 Jul 2025 09:35:24 -0700 Subject: [PATCH 5/5] Migrate K3s Release Artifacts to GHA (#12606) Signed-off-by: Derek Nola --- .drone.yml | 65 ----------------------- .github/workflows/release.yml | 98 ++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 67 deletions(-) diff --git a/.drone.yml b/.drone.yml index e7b364163e4d..9d86283f295d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,28 +49,6 @@ steps: event: - push - tag - -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-amd64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - - name: docker-publish image: plugins/docker settings: @@ -189,27 +167,6 @@ steps: - name: docker path: /var/run/docker.sock -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - - name: docker-publish image: plugins/docker settings: @@ -304,28 +261,6 @@ steps: volumes: - name: docker path: /var/run/docker.sock - -- name: github_binary_release - image: plugins/github-release:linux-arm - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - - name: docker-publish image: plugins/docker:linux-arm settings: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ae7ee913abc..49db1c068d29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: COnfigure image tags + - name: Configure image tags id: tag_config run: | TAG=${GITHUB_REF#refs/tags/} @@ -71,7 +71,8 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/k3s + images: | + ghcr.io/${{ github.repository_owner }}/k3s flavor: latest=false tags: ${{ steps.tag_config.outputs.tag_spec }} @@ -99,3 +100,96 @@ jobs: build-args: | DRONE_TAG=${{ github.ref_name }} + upload-release-airgap: + name: Build Airgap Pkg (${{ matrix.arch }}) + runs-on: ubuntu-latest # Runs on standard runner, docker pulls with --platform + permissions: + contents: write # Needed to update release with assets + strategy: + matrix: + arch: [amd64, arm64, arm] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install Dependencies + run: sudo apt-get update -y && sudo apt-get install -y zstd pigz + + - name: Create Airgap Package (${{ matrix.arch }}) + run: | + mkdir -p ./dist/artifacts + ./scripts/package-airgap ${{ matrix.arch }} + + - name: Caculate Airgap sha256sum + run: sha256sum dist/artifacts/k3s-airgap-images-${{ matrix.arch }}* | sed 's|dist/artifacts/||' > dist/artifacts/k3s-airgap-images-${{ matrix.arch }}.sha256sum + + - name: Upload Airgap sha256sum + uses: actions/upload-artifact@v4 + with: + name: k3s-airgap-images-${{ matrix.arch }}.sha256sum + path: dist/artifacts/k3s-airgap-images-${{ matrix.arch }}.sha256sum + + - name: Upload k3s-images.txt to Release + uses: softprops/action-gh-release@v2 + # This action is recommended by GITHUB, they don't support a first party action for releases + # See https://github.com/actions/create-release?tab=readme-ov-file#github-action---releases-api + if: ${{ matrix.arch == 'amd64' }} + with: + files: | + dist/artifacts/k3s-images.txt + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Airgap Assets to Release + uses: softprops/action-gh-release@v2 + with: + files: | + dist/artifacts/k3s-airgap-images* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + upload-release-assets: + name: Prepare and Upload Release Assets + permissions: + contents: write # Needed to update release with assets + runs-on: ubuntu-latest + needs: [build-amd64, build-arm64, build-arm, upload-release-airgap] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: "Download Binaries and Airgap sha256sum" + uses: actions/download-artifact@v4 + with: + pattern: k3s* + path: ./dist/artifacts + merge-multiple: true + + - name: "Combine and format sha256sum files" + run: | + for arch in amd64 arm64 arm; do + output_file="./dist/artifacts/sha256sum-${arch}.txt" + cat ./dist/artifacts/k3s-airgap-images-$arch*.sha256sum >> "$output_file" + rm ./dist/artifacts/k3s-airgap-images-$arch*.sha256sum + if [[ "$arch" == "amd64" ]]; then + cat ./dist/artifacts/k3s.sha256sum >> "$output_file" + rm ./dist/artifacts/k3s.sha256sum # Remove the original file to avoid uploading it + else + cat ./dist/artifacts/k3s-${arch}.sha256sum >> "$output_file" + rm ./dist/artifacts/k3s-${arch}.sha256sum # Remove the original file to avoid uploading it + fi + done + + - name: Upload Assets to Release + uses: softprops/action-gh-release@v2.2.1 + with: + files: | + dist/artifacts/k3s* + dist/artifacts/sha256sum* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +