Release v2.0.2

.github/workflows/integration.yaml

@@ -16,39 +16,80 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-api-docs:
+  code-check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
-      - name: The API documentation hasn't changed
+      - name: Run go mod tidy
         run: |
-          make build-api-docs
-          if ! git diff --quiet -- docs/api-docs.md; then
-            echo "Need to re-run 'make build-api-docs' and commit the changes"
-            git diff -- docs/api-docs.md;
+          go mod tidy
+          if ! git diff --quiet; then
+            echo "Please run 'go mod tidy' and commit the changes."
+            git diff
             false
           fi
 
-  build-sparkctl:
+      - name: Generate code
+        run: |
+          make generate
+          if ! git diff --quiet; then
+            echo "Need to re-run 'make generate' and commit the changes."
+            git diff
+            false
+          fi
+
+      - name: Run go fmt check
+        run: |
+          make go-fmt
+          if ! git diff --quiet; then
+            echo "Need to re-run 'make go-fmt' and commit the changes."
+            git diff
+            false
+          fi
+
+      - name: Run go vet check
+        run: |
+          make go-vet
+          if ! git diff --quiet; then
+            echo "Need to re-run 'make go-vet' and commit the changes."
+            git diff
+            false
+          fi
+
+      - name: Run golangci-lint
+        run: |
+          make go-lint
+
+  build-api-docs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
-      - name: build sparkctl
-        run: make build-sparkctl
+      - name: Build API docs
+        run: |
+          make build-api-docs
+          if ! git diff --quiet; then
+            echo "Need to re-run 'make build-api-docs' and commit the changes."
+            git diff
+            false
+          fi
 
   build-spark-operator:
     runs-on: ubuntu-latest
@@ -63,18 +104,28 @@ jobs:
         with:
           go-version-file: go.mod
 
-      - name: Run go fmt check
-        run: make go-fmt
-
-      - name: Run go vet check
-        run: make go-vet
-
-      - name: Run unit tests
+      - name: Run go unit tests
         run: make unit-test
 
       - name: Build Spark operator
         run: make build-operator
 
+  build-sparkctl:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Build sparkctl
+        run: make build-sparkctl
+
   build-helm-chart:
     runs-on: ubuntu-latest
     steps:
@@ -103,6 +154,18 @@ jobs:
       - name: Set up chart-testing
         uses: helm/[email protected]
 
+      - name: Generate manifests
+        run: |
+          make manifests
+          if ! git diff --quiet; then
+            echo "Need to re-run 'make manifests' and commit the changes."
+            git diff
+            false
+          fi
+
+      - name: Detect CRDs drift between chart and manifest
+        run: make detect-crds-drift
+
       - name: Run chart-testing (list-changed)
         id: list-changed
         env:
@@ -119,22 +182,18 @@ jobs:
           BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
         run: ct lint --check-version-increment=false --target-branch $BRANCH
 
-      - name: Detect CRDs drift between chart and manifest
-        if: steps.list-changed.outputs.changed == 'true'
-        run: make detect-crds-drift
-
       - name: Produce the helm documentation
         if: steps.list-changed.outputs.changed == 'true'
         run: |
           make helm-docs
           if ! git diff --quiet -- charts/spark-operator-chart/README.md; then
-            echo "Need to re-run 'make helm-docs' and commit the changes"
+            echo "Need to re-run 'make helm-docs' and commit the changes."
             false
           fi
 
       - name: setup minikube
         if: steps.list-changed.outputs.changed == 'true'
-        uses: manusa/actions-setup-minikube@v2.11.0
+        uses: manusa/actions-setup-minikube@v2.12.0
         with:
           minikube version: v1.33.0
           kubernetes version: v1.30.0

.golangci.yaml

@@ -1,39 +1,66 @@
 run:
-  deadline: 5m
+  # Timeout for analysis, e.g. 30s, 5m.
+  # Default: 1m
+  timeout: 1m
 
 linters:
+  # Enable specific linters.
+  # https://golangci-lint.run/usage/linters/#enabled-by-default
   enable:
-  - revive
-  - gci
-  - depguard
-  - godot
-  - testifylint
-  - unconvert
+    # Detects places where loop variables are copied.
+    - copyloopvar
+    # Checks for duplicate words in the source code.
+    - dupword
+    # Tool for detection of FIXME, TODO and other comment keywords.
+    # - godox
+    # Check import statements are formatted according to the 'goimport' command.
+    - goimports
+    # Enforces consistent import aliases.
+    - importas
+    # Find code that shadows one of Go's predeclared identifiers.
+    - predeclared
+    # Check that struct tags are well aligned.
+    - tagalign
+    # Remove unnecessary type conversions.
+    - unconvert
+    # Checks Go code for unused constants, variables, functions and types.
+    - unused
 
 issues:
-  exclude-rules:
-  # Disable errcheck linter for test files.
-  - path: _test.go
-    linters:
-    - errcheck
+  # Which dirs to exclude: issues from them won't be reported.
+  # Can use regexp here: `generated.*`, regexp is applied on full path,
+  # including the path prefix if one is set.
+  # Default dirs are skipped independently of this option's value (see exclude-dirs-use-default).
+  # "/" will be replaced by current OS file path separator to properly work on Windows.
+  # Default: []
+  exclude-dirs:
+    - sparkctl
+  # Maximum issues count per one linter.
+  # Set to 0 to disable.
+  # Default: 50
+  max-issues-per-linter: 50
+  # Maximum count of issues with the same text.
+  # Set to 0 to disable.
+  # Default: 3
+  max-same-issues: 3
 
 linters-settings:
-  gci:
-    sections:
-      - standard
-      - default
-      - prefix(github.com/kubeflow/spark-operator)
-  depguard:
-    Main:
-      files:
-        - $all
-        - "!$test"
-      listMode: Lax
-      deny:
-        reflect: Please don't use reflect package
-    Test:
-      files:
-        - $test
-      listMode: Lax
-      deny:
-        reflect: Please don't use reflect package
+  importas:
+    # List of aliases
+    alias:
+    - pkg: k8s.io/api/admissionregistration/v1
+      alias: admissionregistrationv1
+    - pkg: k8s.io/api/apps/v1
+      alias: appsv1
+    - pkg: k8s.io/api/batch/v1
+      alias: batchv1
+    - pkg: k8s.io/api/core/v1
+      alias: corev1
+    - pkg: k8s.io/api/extensions/v1beta1
+      alias: extensionsv1beta1
+    - pkg: k8s.io/api/networking/v1
+      alias: networkingv1
+    - pkg: k8s.io/apimachinery/pkg/apis/meta/v1
+      alias: metav1
+    - pkg: sigs.k8s.io/controller-runtime
+      alias: ctrl

Dockerfile

@@ -20,10 +20,6 @@ FROM golang:1.23.1 AS builder
 
 WORKDIR /workspace
 
-RUN apt-get update \
-    && apt-get install -y libcap2-bin \
-    && rm -rf /var/lib/apt/lists/*
-
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=bind,source=go.mod,target=go.mod \
     --mount=type=bind,source=go.sum,target=go.sum \
@@ -36,7 +32,6 @@ ARG TARGETARCH
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target="/root/.cache/go-build" \
     CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on make build-operator
-RUN setcap 'cap_net_bind_service=+ep' /workspace/bin/spark-operator
 
 FROM ${SPARK_IMAGE}
 
@@ -46,9 +41,9 @@ RUN apt-get update \
     && apt-get install -y tini \
     && rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /etc/k8s-webhook-server/serving-certs && \
+RUN mkdir -p /etc/k8s-webhook-server/serving-certs /home/spark && \
     chmod -R g+rw /etc/k8s-webhook-server/serving-certs && \
-    chown -R spark /etc/k8s-webhook-server/serving-certs
+    chown -R spark /etc/k8s-webhook-server/serving-certs /home/spark
 
 USER spark
 

Makefile

@@ -55,7 +55,7 @@ KIND_VERSION ?= v0.23.0
 ENVTEST_VERSION ?= release-0.18
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION ?= 1.29.3
-GOLANGCI_LINT_VERSION ?= v1.57.2
+GOLANGCI_LINT_VERSION ?= v1.61.0
 GEN_CRD_API_REFERENCE_DOCS_VERSION ?= v0.3.0
 HELM_VERSION ?= v3.15.3
 HELM_UNITTEST_VERSION ?= 0.5.1
@@ -136,13 +136,13 @@ go-vet: ## Run go vet against code.
 	@echo "Running go vet..."
 	go vet ./...
 
-.PHONY: lint
-lint: golangci-lint ## Run golangci-lint linter.
+.PHONY: go-lint
+go-lint: golangci-lint ## Run golangci-lint linter.
 	@echo "Running golangci-lint run..."
 	$(GOLANGCI_LINT) run
 
-.PHONY: lint-fix
-lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes.
+.PHONY: go-lint-fix
+go-lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes.
 	@echo "Running golangci-lint run --fix..."
 	$(GOLANGCI_LINT) run --fix
 

VERSION

@@ -1 +1 @@
-v2.0.1
+v2.0.2

api/v1beta1/sparkapplication_types.go

@@ -70,8 +70,8 @@ type SparkApplicationSpec struct {
 	// spark-submit.
 	// Optional.
 	SparkConf map[string]string `json:"sparkConf,omitempty"`
-	// HadoopConf carries user-specified Hadoop configuration properties as they would use the  the "--conf" option
-	// in spark-submit.  The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop
+	// HadoopConf carries user-specified Hadoop configuration properties as they would use the "--conf" option
+	// in spark-submit. The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop
 	// configuration properties.
 	// Optional.
 	HadoopConf map[string]string `json:"hadoopConf,omitempty"`

api/v1beta2/defaults.go

@@ -54,7 +54,6 @@ func SetSparkApplicationDefaults(app *SparkApplication) {
 }
 
 func setDriverSpecDefaults(spec *DriverSpec, sparkConf map[string]string) {
-
 	if _, exists := sparkConf["spark.driver.cores"]; !exists && spec.Cores == nil {
 		spec.Cores = new(int32)
 		*spec.Cores = 1

api/v1beta2/defaults_test.go

@@ -121,7 +121,6 @@ func TestSetSparkApplicationDefaultsOnFailureRestartPolicyShouldSetDefaultValueF
 }
 
 func TestSetSparkApplicationDefaultsDriverSpecDefaults(t *testing.T) {
-
 	//Case1: Driver config not set.
 	app := &SparkApplication{
 		Spec: SparkApplicationSpec{},

api/v1beta2/doc.go

@@ -15,7 +15,6 @@ limitations under the License.
 */
 
 // +k8s:deepcopy-gen=package,register
-// go:generate controller-gen crd:trivialVersions=true paths=. output:dir=.
 
 // Package v1beta2 is the v1beta2 version of the API.
 // +groupName=sparkoperator.k8s.io

api/v1beta2/sparkapplication_types.go

@@ -71,8 +71,8 @@ type SparkApplicationSpec struct {
 	// spark-submit.
 	// +optional
 	SparkConf map[string]string `json:"sparkConf,omitempty"`
-	// HadoopConf carries user-specified Hadoop configuration properties as they would use the  the "--conf" option
-	// in spark-submit.  The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop
+	// HadoopConf carries user-specified Hadoop configuration properties as they would use the "--conf" option
+	// in spark-submit. The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop
 	// configuration properties.
 	// +optional
 	HadoopConf map[string]string `json:"hadoopConf,omitempty"`

charts/spark-operator-chart/Chart.yaml

@@ -20,9 +20,9 @@ name: spark-operator
 
 description: A Helm chart for Spark on Kubernetes operator.
 
-version: 2.0.1
+version: 2.0.2
 
-appVersion: 2.0.1
+appVersion: 2.0.2
 
 keywords:
 - apache spark