diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de633f7c..50296286 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,9 +26,9 @@ env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: '0' CARGO_PROFILE_DEV_DEBUG: '0' - RUST_TOOLCHAIN_VERSION: "1.85.0" + RUST_TOOLCHAIN_VERSION: "1.87.0" RUST_NIGHTLY_TOOLCHAIN_VERSION: "nightly-2025-05-26" - PYTHON_VERSION: "3.12" + PYTHON_VERSION: "3.13" RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" RUST_LOG: "info" @@ -42,7 +42,7 @@ jobs: RUSTC_BOOTSTRAP: 1 steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 + uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest @@ -50,10 +50,10 @@ jobs: with: persist-credentials: false submodules: recursive - - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 with: key: udeps cache-all-crates: "true" @@ -114,7 +114,7 @@ jobs: with: persist-credentials: false submodules: recursive - - uses: EmbarkStudios/cargo-deny-action@34899fc7ba81ca6268d5947a7a16b4649013fea1 # v2.0.11 + - uses: EmbarkStudios/cargo-deny-action@30f817c6f72275c6d54dc744fbca09ebc958599f # v2.0.12 with: command: check ${{ matrix.checks }} @@ -126,7 +126,7 @@ jobs: with: persist-credentials: false submodules: recursive - - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_NIGHTLY_TOOLCHAIN_VERSION }} components: rustfmt @@ -139,7 +139,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 + uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest @@ -147,11 +147,11 @@ jobs: with: persist-credentials: false submodules: recursive - - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} components: clippy - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 with: key: clippy cache-all-crates: "true" @@ -178,18 +178,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 + uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} components: rustfmt - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 with: key: doc cache-all-crates: "true" @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 + uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest @@ -209,10 +209,10 @@ jobs: with: persist-credentials: false submodules: recursive - - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 with: key: test cache-all-crates: "true" @@ -261,7 +261,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 + uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest @@ -274,10 +274,10 @@ jobs: with: version: v3.16.1 - name: Set up cargo - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 with: key: charts cache-all-crates: "true" @@ -332,7 +332,7 @@ jobs: IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }} steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 + uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ${{ matrix.runner }} @@ -340,8 +340,8 @@ jobs: with: persist-credentials: false submodules: recursive - - uses: cachix/install-nix-action@17fe5fb4a23ad6cbbe47d6b3f359611ad276644c # v31.4.0 - - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 + - uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31.4.1 + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b with: toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} components: rustfmt @@ -376,9 +376,9 @@ jobs: # Recreate charts and publish charts and docker image. - name: Install cosign - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 - name: Install syft - uses: anchore/sbom-action/download-syft@e11c554f704a0b820cbf8c51673f6945e0731532 # v0.20.0 + uses: anchore/sbom-action/download-syft@cee1b8e05ae5b2593a75e197229729eabaa9f8ec # v0.20.2 - name: Build Docker image and Helm chart run: | # Installing helm and yq on ubicloud-standard-8-arm only @@ -421,7 +421,7 @@ jobs: OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build" steps: - name: Install cosign - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 321a0387..767f57e9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -86,7 +86,7 @@ jobs: - name: Run Integration Test id: test - uses: stackabletech/actions/run-integration-test@5901c3b1455488820c4be367531e07c3c3e82538 # v0.4.0 + uses: stackabletech/actions/run-integration-test@4483641a7e24057bd2ba51cb4c3f2f0010ad21b7 # v0.8.4 with: test-platform: ${{ env.TEST_PLATFORM }}-${{ env.TEST_ARCHITECTURE }} test-run: ${{ env.TEST_RUN }} @@ -117,3 +117,21 @@ jobs: } ] } + # TODO: Update to version 2.1.0. This could look something like the following. + # The workflow is currently not in use, testing that the new version still works imposes effort. + # So I left it as a future exercise, but saved the current state. + # + # uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + # with: + # method: chat.postMessage + # token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }} + # payload: | + # channel: "C07UYJYSMSN" # notifications-integration-tests + # text: "Integration Test *${{ github.repository }}* failed" + # attachments: + # - pretext: "Started at ${{ steps.test.outputs.start-time }}, failed at ${{ steps.test.outputs.end-time }}" + # color: "#aa0000" + # actions: + # - type: button + # text: Go to integration test run + # url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml index 35dc6ac7..a781f1e6 100644 --- a/.github/workflows/pr_pre-commit.yaml +++ b/.github/workflows/pr_pre-commit.yaml @@ -7,10 +7,10 @@ on: env: CARGO_TERM_COLOR: always - NIX_PKG_MANAGER_VERSION: "2.28.3" + NIX_PKG_MANAGER_VERSION: "2.30.0" RUST_TOOLCHAIN_VERSION: "nightly-2025-05-26" HADOLINT_VERSION: "v2.12.0" - PYTHON_VERSION: "3.12" + PYTHON_VERSION: "3.13" jobs: pre-commit: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1bc58c7..d8f954a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: # If you do not, you will need to delete the cached ruff binary shown in the # error message - repo: https://github.com/astral-sh/ruff-pre-commit - rev: d19233b89771be2d89273f163f5edc5a39bbc34a # 0.11.12 + rev: 0b19ef1fd6ad680ed7752d6daba883ce1265a6de # 0.12.2 hooks: # Run the linter. - id: ruff-check diff --git a/Makefile b/Makefile index fd1e8851..02bad668 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ check-kubernetes: run-dev: check-nix check-kubernetes kubectl apply -f deploy/stackable-operators-ns.yaml - nix run --extra-experimental-features "nix-command flakes" -f. tilt -- up --port 5430 --namespace stackable-operators + nix run --extra-experimental-features "nix-command flakes" -f. tilt -- up --port 5444 --namespace stackable-operators stop-dev: check-nix check-kubernetes nix run --extra-experimental-features "nix-command flakes" -f. tilt -- down diff --git a/deploy/helm/trino-operator/templates/deployment.yaml b/deploy/helm/trino-operator/templates/deployment.yaml index 2429e0c3..9056e705 100644 --- a/deploy/helm/trino-operator/templates/deployment.yaml +++ b/deploy/helm/trino-operator/templates/deployment.yaml @@ -42,11 +42,26 @@ spec: - mountPath: /etc/stackable/{{ include "operator.appname" . }}/config-spec name: config-spec env: + # The following env vars are passed as clap (think CLI) arguments to the operator. + # They are picked up by clap using the structs defied in the operator. + # (which is turn pulls in https://github.com/stackabletech/operator-rs/blob/main/crates/stackable-operator/src/cli.rs) + # You can read there about the expected values and purposes. + + # Sometimes products need to know the operator image, e.g. the opa-bundle-builder OPA + # sidecar uses the operator image. - name: OPERATOR_IMAGE # Tilt can use annotations as image paths, but not env variables valueFrom: fieldRef: fieldPath: metadata.annotations['internal.stackable.tech/image'] + + # Operators need to know the node name they are running on, to e.g. discover the + # Kubernetes domain name from the kubelet API. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.kubernetesClusterDomain }} - name: KUBERNETES_CLUSTER_DOMAIN value: {{ .Values.kubernetesClusterDomain | quote }} diff --git a/docker/Dockerfile b/docker/Dockerfile index e632601e..52c24ac8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,6 +26,7 @@ ARG RELEASE="1" # These are chosen at random and are this high on purpose to have very little chance to clash with an existing user or group on the host system ARG STACKABLE_USER_GID="574654813" ARG STACKABLE_USER_UID="782252253" +ARG STACKABLE_USER_NAME="stackable" # Sets the default shell to Bash with strict error handling and robust pipeline processing. # "-e": Exits immediately if a command exits with a non-zero status @@ -95,6 +96,12 @@ RUN <//archive/.tar.gz" } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 9ae738fb..a9d21ba2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ # DO NOT EDIT, this file is generated by operator-templating [toolchain] -channel = "1.85.0" +channel = "1.87.0" profile = "default" diff --git a/scripts/run-tests b/scripts/run-tests index 31b8e7ae..904f5811 100755 --- a/scripts/run-tests +++ b/scripts/run-tests @@ -353,22 +353,7 @@ def run_tests(test: str, parallel: int, namespace: str, skip_delete: bool) -> No if namespace: kuttl_cmd.extend(["--namespace", namespace]) # kuttl doesn't create the namespace so we need to do it ourselves - create_ns_cmd = ["kubectl", "create", "namespace", namespace] - try: - logging.debug(f"Running : {create_ns_cmd}") - subprocess.run( - create_ns_cmd, - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError as e: - stderr = e.stderr.decode("utf-8") - # If the namespace already exists, this will fail and we ignore the - # error. If it fails for any other reason, we raise an exception. - if "already exists" not in stderr: - logging.error(stderr) - logging.error("namespace creation failed") - raise TestRunnerException() + ensure_namespace_exists(namespace) logging.debug(f"Running : {kuttl_cmd}") @@ -382,6 +367,58 @@ def run_tests(test: str, parallel: int, namespace: str, skip_delete: bool) -> No raise TestRunnerException() +def ensure_namespace_exists(namespace: str) -> None: + """ + Ensure the specified namespace exists, creating it if necessary. + + This function handles various permission scenarios: + - If the namespace already exists, it does nothing + - If it doesn't exist and we have permission, it creates it + - If we don't have permission to create/check namespaces, it logs a warning + and assumes the namespace exists or will be created externally (useful for OpenShift) + + Examples of (permission) errors we handle: + - Error from server (Forbidden): namespaces is forbidden: User "developer" cannot create resource "namespaces" in API group "" at the cluster scope + - Error from server (Forbidden): namespaces "foobar123" is forbidden: User "developer" cannot get resource "namespaces" in API group "" in the namespace "foobar123" + - Error from server (AlreadyExists): namespaces "kuttl-test-finer-caiman" already exists + """ + # First check if the namespace already exists + check_ns_cmd = ["kubectl", "get", "namespace", namespace] + try: + logging.debug(f"Checking if namespace exists: {check_ns_cmd}") + subprocess.run( + check_ns_cmd, + check=True, + capture_output=True, + ) + logging.debug(f"Namespace '{namespace}' already exists") + except subprocess.CalledProcessError: + # Namespace doesn't exist, try to create it + create_ns_cmd = ["kubectl", "create", "namespace", namespace] + try: + logging.debug(f"Creating namespace: {create_ns_cmd}") + subprocess.run( + create_ns_cmd, + check=True, + capture_output=True, + ) + logging.debug(f"Successfully created namespace '{namespace}'") + except subprocess.CalledProcessError as e: + stderr = e.stderr.decode("utf-8") + if "already exists" in stderr: + logging.debug( + f"Namespace '{namespace}' already exists (race condition)" + ) + elif "forbidden" in stderr.lower(): + logging.warning( + f"No permission to create namespace '{namespace}', assuming it exists or will be created externally" + ) + else: + logging.error(stderr) + logging.error("namespace creation failed") + raise TestRunnerException() + + def main(argv) -> int: ret = 0 try: