feat(cli): add package and node management commands with lifecycle controls (#123)

Add comprehensive CLI commands for managing Skyhook packages and nodes:

Package Commands:
- `package rerun`: Force re-execution of packages on specific nodes
  - Support for stage-specific re-runs (apply, config, interrupt, post-interrupt)
  - Node matching via exact names or regex patterns
- `package status`: Query package status across the cluster
- `package logs`: Retrieve package execution logs with follow/tail support

Node Commands:
- `node list`: List all nodes with Skyhook status
- `node status`: Display detailed status for specific nodes
- `node ignore`: Add/remove ignore label to pause operations on nodes
- `node reset`: Reset node state for a Skyhook

Lifecycle Commands:
- `pause`: Pause Skyhook reconciliation temporarily
- `resume`: Resume paused Skyhook operations
- `disable`: Disable a Skyhook completely
- `enable`: Re-enable a disabled Skyhook

Also includes:
- Comprehensive unit tests with K8s dynamic client mocks
- CLI e2e test suite using chainsaw (lifecycle, node, package tests)
- CI integration for CLI tests in operator-ci workflow
- Shared utilities for node matching, label management, and patch-based updates

Author: t0mmylam

.github/workflows/operator-ci.yaml

@@ -61,6 +61,10 @@ jobs:
             test-suite: deployment-policy
             kind-config: k8s-tests/chainsaw/deployment-policy/kind-config.yaml
             make-target: deployment-policy-tests
+          # CLI e2e tests on K8s 1.34 only
+          - k8s-version: "1.34.0"
+            test-suite: cli-e2e
+            make-target: cli-e2e-tests
       fail-fast: false  # Continue testing other versions if one fails
     name: ${{ matrix.test-suite }}-tests (k8s-${{ matrix.k8s-version }})
     steps:
@@ -118,6 +122,9 @@ jobs:
           if [ "${{ matrix.test-suite }}" = "e2e" ]; then
             make setup-kind-cluster
             make test
+          elif [ "${{ matrix.test-suite }}" = "cli-e2e" ]; then
+            make setup-kind-cluster
+            make ${{ matrix.make-target }}
           else
             make ${{ matrix.make-target }}
           fi

k8s-tests/chainsaw/cli/lifecycle/assert-disabled.yaml

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+  annotations:
+    skyhook.nvidia.com/disable: "true"

k8s-tests/chainsaw/cli/lifecycle/assert-enabled.yaml

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+  # Note: Chainsaw will verify that disable annotation is NOT present

k8s-tests/chainsaw/cli/lifecycle/assert-paused-and-disabled.yaml

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+  annotations:
+    skyhook.nvidia.com/pause: "true"
+    skyhook.nvidia.com/disable: "true"

k8s-tests/chainsaw/cli/lifecycle/assert-paused.yaml

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+  annotations:
+    skyhook.nvidia.com/pause: "true"

k8s-tests/chainsaw/cli/lifecycle/assert-resumed.yaml

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+  # Note: Chainsaw will verify that pause annotation is NOT present
+  # by checking the resource exists without that annotation

k8s-tests/chainsaw/cli/lifecycle/assert-still-disabled.yaml

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+  annotations:
+    # pause annotation should be removed, but disable should remain
+    skyhook.nvidia.com/disable: "true"

k8s-tests/chainsaw/cli/lifecycle/chainsaw-test.yaml

@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: cli-lifecycle
+spec:
+  description: |
+    Test all CLI lifecycle commands:
+    - pause: pauses a Skyhook from processing
+    - resume: resumes a paused Skyhook
+    - disable: disables a Skyhook completely
+    - enable: enables a disabled Skyhook
+  timeouts:
+    assert: 60s
+    exec: 30s
+  steps:
+  # Step 1: Create a Skyhook
+  - name: setup-skyhook
+    try:
+    - apply:
+        file: skyhook.yaml
+    - script:
+        timeout: 30s
+        content: |
+          echo "Waiting for Skyhook to be created..."
+          for i in $(seq 1 15); do
+            if kubectl get skyhook cli-lifecycle-test 2>/dev/null; then
+              echo "Skyhook created"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Timeout waiting for Skyhook"
+          exit 1
+
+  # Step 2: Test pause command
+  - name: test-pause
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          ../../../../operator/bin/skyhook pause cli-lifecycle-test --confirm
+    - assert:
+        file: assert-paused.yaml
+
+  # Step 3: Test resume command
+  - name: test-resume
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          ../../../../operator/bin/skyhook resume cli-lifecycle-test
+    - assert:
+        file: assert-resumed.yaml
+
+  # Step 4: Test disable command
+  - name: test-disable
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          ../../../../operator/bin/skyhook disable cli-lifecycle-test --confirm
+    - assert:
+        file: assert-disabled.yaml
+
+  # Step 5: Test enable command
+  - name: test-enable
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          ../../../../operator/bin/skyhook enable cli-lifecycle-test
+    - assert:
+        file: assert-enabled.yaml
+
+  # Step 6: Test pause and disable together
+  - name: test-pause-and-disable
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          ../../../../operator/bin/skyhook pause cli-lifecycle-test --confirm
+          ../../../../operator/bin/skyhook disable cli-lifecycle-test --confirm
+    - assert:
+        file: assert-paused-and-disabled.yaml
+    - script:
+        timeout: 30s
+        content: |
+          # Resume should only remove pause, not disable
+          ../../../../operator/bin/skyhook resume cli-lifecycle-test
+    - assert:
+        file: assert-still-disabled.yaml
+    - script:
+        timeout: 30s
+        content: |
+          # Enable should remove disable
+          ../../../../operator/bin/skyhook enable cli-lifecycle-test
+
+  # Cleanup
+  - name: cleanup
+    try:
+    - script:
+        content: |
+          kubectl delete skyhook cli-lifecycle-test 2>/dev/null || true
+

k8s-tests/chainsaw/cli/lifecycle/skyhook.yaml

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-lifecycle-test
+spec:
+  nodeSelectors:
+    matchLabels:
+      skyhook.nvidia.com/test-node: skyhooke2e
+  packages:
+    test-package:
+      version: "1.0.0"
+      image: ghcr.io/nvidia/skyhook-packages/shellscript
+      configMap:
+        apply.sh: |
+          #!/bin/bash
+          echo "Lifecycle test apply"
+          sleep 2
+

k8s-tests/chainsaw/cli/node/assert-node-ignored.yaml

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Node
+metadata:
+  labels:
+    skyhook.nvidia.com/test-node: skyhooke2e
+    skyhook.nvidia.com/ignore: "true"