Merge pull request #1855 from kube-rs/e2e-deb-distroless

Shift e2e to official rust image with distroless

Author: clux

.dockerignore

@@ -0,0 +1,5 @@
+.git/
+target/
+e2e/Dockerfile
+e2e/target
+justfile

.github/workflows/ci.yml

@@ -227,40 +227,71 @@ jobs:
         tls: [openssl, rustls]
     steps:
       - uses: actions/checkout@v6
-      - uses: actions/cache@v4
+
+      - name: Setup Docker buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+
+      # clux: i can never get the buildkit cache dance seems to work reliably so commenting it out
+      # would like to get this to work long term, but at least the build is pretty fast
+      # - uses: actions/cache@v4
+      #   id: cache
+      #   with:
+      #     path: |
+      #       target
+      #       .cargo/registry
+      #       .cargo/git
+      #     key: docker-cargo4-${{matrix.tls}}-${{ hashFiles('**/Cargo.toml', 'e2e/Dockerfile') }}
+      #     restore-keys: |
+      #       docker-cargo4-${{matrix.tls}}-
+      - name: Create folders on cache miss (blank cache always atm)
+        run: mkdir -p ~/.cargo/{git,registry} && mkdir -p target
+
+      # - name: Inject cache into docker
+      #   uses: reproducible-containers/buildkit-cache-dance@v3
+      #   with:
+      #     dockerfile: e2e/Dockerfile
+      #     builder: ${{ steps.setup-buildx.outputs.name }}
+      #     cache-map: |
+      #       {
+      #         "target": {
+      #           "target": "/app/target",
+      #           "id": "target"
+      #         },
+      #         ".cargo/registry": "/root/.cargo/registry",
+      #         ".cargo/git": "/root/.cargo/git"
+      #       }
+      #     skip-extraction: ${{ steps.cache.outputs.cache-hit }}
+
+      - name: Build and push Docker image
+        id: build
+        uses: docker/build-push-action@v6
         with:
-          path: |
-            ~/.cargo/registry/index
-            ~/.cargo/registry/cache
-            ~/.cargo/git
-            target
-          key: musl-cargo-${{ hashFiles('**/Cargo.toml') }}-${{matrix.tls}}
+          context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=min
+          platforms: linux/amd64
+          push: false
+          load: true
+          file: e2e/Dockerfile
+          tags: clux/kube-e2e:${{ github.sha }}
+          build-args: |
+            FEATURES=latest,${{matrix.tls}}
 
       - uses: nolar/setup-k3d-k3s@v1
         with:
           version: latest
-          # k3d-kube
           k3d-name: kube
           # Used to avoid rate limits when fetching the releases from k3s repo.
           # Anonymous access is limited to 60 requests / hour / worker
           # github-token: ${{ secrets.GITHUB_TOKEN }}
           k3d-args: "--no-lb --no-rollback --k3s-arg --disable=traefik,servicelb,metrics-server@server:*"
-      - name: Compile e2e job against ${{matrix.tls}}
-        run: |
-          mkdir -p ~/.cargo/{git,registry}
-          docker run --rm -t \
-              --mount type=bind,source=${{ github.workspace }},target=/volume \
-              --mount type=bind,source=$HOME/.cargo/registry,target=/root/.cargo/registry \
-              --mount type=bind,source=$HOME/.cargo/git,target=/root/.cargo/git \
-              clux/muslrust:1.86.0-stable \
-              cargo build -p e2e --release --bin=job --features=latest,${{matrix.tls}} -v
-          cp target/x86_64-unknown-linux-musl/release/job e2e/
-
-      - name: Build image
-        run: "docker build -t clux/kube-e2e:${{ github.sha }} e2e/"
       - name: Import image
         run: "k3d image import clux/kube-e2e:${{ github.sha }} --cluster kube"
-      - run: sed -i 's/latest/${{ github.sha }}/g' e2e/deployment.yaml
+      - name: Template the yaml and verify we are testing what we built
+        run: |
+          sed -i 's/latest/${{ github.sha }}/g' e2e/deployment.yaml
+          grep -q "${{ github.sha }}" e2e/deployment.yaml
 
       - name: Create resource
         run: kubectl apply -f e2e/deployment.yaml -n apps

e2e/Dockerfile

@@ -1,3 +1,17 @@
-FROM gcr.io/distroless/static:nonroot
-COPY --chown=nonroot:nonroot ./job /app/
-ENTRYPOINT ["/app/job"]
+# syntax=docker/dockerfile:1
+FROM rust:bookworm AS builder
+ARG FEATURES=""
+WORKDIR /app
+COPY . /app
+# NB: need rw even on fetch because we do not have a lockfile checked in
+RUN --mount=type=bind,rw,target=. \
+    --mount=type=cache,target=/root/.cargo \
+    cargo fetch
+RUN --mount=type=bind,rw,target=. \
+    --mount=type=cache,target=/root/.cargo \
+    --mount=type=cache,target=target,id=target \
+    cargo install --path=e2e --features=${FEATURES} --bin=job
+
+FROM gcr.io/distroless/cc-debian12
+COPY --from=builder /usr/local/cargo/bin/job /app/
+CMD ["/app/job"]

justfile

@@ -73,9 +73,9 @@ e2e-mink8s:
   #cargo run -p e2e --bin boot --features=rustls,mk8sv
 
 e2e-incluster features:
-  just e2e-job-musl {{features}}
-  docker build -t clux/kube-e2e:{{VERSION}} e2e/
-  k3d image import clux/kube-e2e:{{VERSION}} --cluster main
+  docker build --build-arg FEATURES="{{features}}" \
+    -t clux/kube-e2e:{{VERSION}} . -f e2e/Dockerfile
+  k3d image import clux/kube-e2e:{{VERSION}} -c=$(k3d cluster list -ojson |jq '.[0].name' -r)
   sed -i 's/latest/{{VERSION}}/g' e2e/deployment.yaml
   kubectl apply -f e2e/deployment.yaml
   sed -i 's/{{VERSION}}/latest/g' e2e/deployment.yaml
@@ -84,14 +84,8 @@ e2e-incluster features:
   kubectl wait --for=condition=complete job/e2e -n apps --timeout=50s || kubectl logs -f job/e2e -n apps
   kubectl get all -n apps
   kubectl wait --for=condition=complete job/e2e -n apps --timeout=10s || kubectl get pods -n apps | grep e2e | grep Completed
-e2e-job-musl features:
-  #!/usr/bin/env bash
-  docker run \
-    -v cargo-cache:/root/.cargo/registry \
-    -v "$PWD:/volume" -w /volume \
-    --rm -it clux/muslrust:1.86.0-stable cargo build --release --features={{features}} -p e2e
-  cp target/x86_64-unknown-linux-musl/release/job e2e/job
-  chmod +x e2e/job
+  @echo "need to not commit a sha in the deployment.yaml"
+  rg "latest" e2e/deployment.yaml -q
 
 k3d:
   k3d cluster create main --servers 1 --registry-create main --image rancher/k3s:v1.27.3-k3s1 \