Merge pull request #1718 from Altinity/0.25.0

0.25.0

Author: sunsingerus

.github/workflows/check_helm.yaml

@@ -21,14 +21,19 @@ jobs:
         with:
           driver: docker
           container-runtime: containerd
-          kubernetes-version: v1.31.0
+          kubernetes-version: v1.33.0
           cpus: max
           memory: max
 
+      - uses: gabe565/setup-helm-docs-action@v1
+        with:
+          version: latest
+
       - name: Local Helm Test
         run: |
           set -xe
           minikube kubectl create ns test
+          ./dev/generate_helm_chart.sh
           helm install -n test test-operator ./deploy/helm/clickhouse-operator/
           sleep 10
           workflow_count=$(kubectl get all --all-namespaces | grep -c "test-operator")
@@ -41,4 +46,8 @@ jobs:
           if [ "$configmap_count" -lt 9 ]; then
             echo "Error: ConfigMap count is less than 9. Current count: $configmap_count"
             exit 1
-          fi
+          fi
+          kubectl apply -n test -f ./docs/chi-examples/01-simple-layout-01-1shard-1repl.yaml
+          kubectl wait -n test --for=create chi/simple-01 --timeout=60s
+          kubectl wait -n test --for=jsonpath='{.status.status}'=Completed chi/simple-01 --timeout=1m
+          kubectl wait -n test --for=condition=Ready pod/chi-simple-01-simple-0-0-0

.github/workflows/run_tests.yaml

@@ -47,7 +47,7 @@ jobs:
         with:
           driver: docker
           container-runtime: containerd
-          kubernetes-version: v1.31.0
+          kubernetes-version: v1.31.2
           cpus: max
           memory: max
 
@@ -116,7 +116,7 @@ jobs:
         with:
           name: testflows-report
           path: |
-            tests/report.html
+            tests/*.html
           if-no-files-found: error
           retention-days: 90
 

README.md

@@ -1,6 +1,6 @@
-# Altinity Kubernetes Operator for ClickHouse®
+# Altinity® Kubernetes Operator for ClickHouse®
 
-Altinity Kubernetes Operator for ClickHouse creates, configures and manages ClickHouse clusters running on Kubernetes.
+The Altinity Kubernetes Operator for ClickHouse creates, configures and manages ClickHouse clusters running on Kubernetes.
 
 [![Build Master](https://github.com/Altinity/clickhouse-operator/actions/workflows/build_master.yaml/badge.svg)](https://github.com/Altinity/clickhouse-operator/actions/workflows/build_master.yaml)
 [![GitHub release](https://img.shields.io/github/v/release/altinity/clickhouse-operator?include_prereleases)](https://img.shields.io/github/v/release/altinity/clickhouse-operator?include_prereleases)

config/config-dev.yaml

@@ -119,11 +119,15 @@ clickhouse:
           - settings/remote_servers/*: "no"
           - settings/user_directories/*: "no"
 
+          # these settings should not lead to pod restarts
+          - settings/display_secrets_in_show_and_select: "no"
+
           - zookeeper/*: "yes"
 
           - files/*.xml: "yes"
           - files/config.d/*.xml: "yes"
           - files/config.d/*dict*.xml: "no"
+          - files/config.d/*no_restart*: "no"
 
           # exceptions in default profile
           - profiles/default/background_*_pool_size: "yes"
@@ -147,7 +151,6 @@ clickhouse:
     # These credentials are used for:
     #   1. Metrics requests
     #   2. Schema maintenance
-    #   3. DROP DNS CACHE
     # User with these credentials can be specified in additional ClickHouse .xml config files,
     # located in 'clickhouse.configuration.file.path.user' folder
     username: "clickhouse_operator"
@@ -173,9 +176,61 @@ clickhouse:
     # Specified in seconds.
     timeouts:
       # Timout to setup connection from the operator to ClickHouse instances. In seconds.
-      connect: 2
+      connect: 1
       # Timout to perform SQL query from the operator to ClickHouse instances. In seconds.
-      query: 5
+      query: 4
+
+  ################################################
+  ##
+  ## Addons specifies additional configuration sections
+  ## Should it be called something like "templates"?
+  ##
+  ################################################
+  addons:
+    rules:
+      - version: "*"
+        spec:
+          configuration:
+            users:
+            profiles:
+            quotas:
+            settings:
+            files:
+      - version: ">= 23.3"
+        spec:
+          configuration:
+            ###
+            ### users.d is global while description depends on CH version which may vary on per-host basis
+            ### In case of global-ness this may be better to implement via auto-templates
+            ###
+            ### As a solution, this may be applied on the whole cluster based on any of its hosts
+            ###
+            ### What to do when host is just created? CH version is not known prior to CH started and user config is required before CH started.
+            ### We do not have any info about the cluster on initial creation
+            ###
+            users:
+              "{clickhouseOperatorUser}/access_management": 1
+              "{clickhouseOperatorUser}/named_collection_control": 1
+              "{clickhouseOperatorUser}/show_named_collections": 1
+              "{clickhouseOperatorUser}/show_named_collections_secrets": 1
+            profiles:
+            quotas:
+            settings:
+            files:
+
+      - version: ">= 23.5"
+        spec:
+          configuration:
+            users:
+            profiles:
+              clickhouse_operator/format_display_secrets_in_show_and_select: 1
+            quotas:
+            settings:
+              ##
+              ## this may be added on per-host basis into host's conf.d folder
+              ##
+              display_secrets_in_show_and_select: 1
+            files:
 
   #################################################
   ##
@@ -250,7 +305,7 @@ reconcile:
   # Reconcile runtime settings
   runtime:
     # Max number of concurrent CHI reconciles in progress
-    reconcileCHIsThreadsNumber: 10
+    reconcileCHIsThreadsNumber: 1
 
     # The operator reconciles shards concurrently in each CHI with the following limitations:
     #   1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently
@@ -261,7 +316,7 @@ reconcile:
     # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage
 
     # Max number of concurrent shard reconciles within one CHI in progress
-    reconcileShardsThreadsNumber: 5
+    reconcileShardsThreadsNumber: 1
     # Max percentage of concurrent shard reconciles within one CHI in progress
     reconcileShardsMaxConcurrencyPercent: 50
 
@@ -306,6 +361,10 @@ reconcile:
       exclude: true
       queries: true
       include: false
+      replicas:
+        all: no
+        new: yes
+        delay: 10
 
 ################################################
 ##

config/config.yaml

@@ -119,6 +119,9 @@ clickhouse:
           - settings/remote_servers/*: "no"
           - settings/user_directories/*: "no"
 
+          # these settings should not lead to pod restarts
+          - settings/display_secrets_in_show_and_select: "no"
+
           - zookeeper/*: "yes"
 
           - files/*.xml: "yes"
@@ -177,6 +180,58 @@ clickhouse:
       # Timout to perform SQL query from the operator to ClickHouse instances. In seconds.
       query: 4
 
+  ################################################
+  ##
+  ## Addons specifies additional configuration sections
+  ## Should it be called something like "templates"?
+  ##
+  ################################################
+  addons:
+    rules:
+      - version: "*"
+        spec:
+          configuration:
+            users:
+            profiles:
+            quotas:
+            settings:
+            files:
+      - version: ">= 23.3"
+        spec:
+          configuration:
+            ###
+            ### users.d is global while description depends on CH version which may vary on per-host basis
+            ### In case of global-ness this may be better to implement via auto-templates
+            ###
+            ### As a solution, this may be applied on the whole cluster based on any of its hosts
+            ###
+            ### What to do when host is just created? CH version is not known prior to CH started and user config is required before CH started.
+            ### We do not have any info about the cluster on initial creation
+            ###
+            users:
+              "{clickhouseOperatorUser}/access_management": 1
+              "{clickhouseOperatorUser}/named_collection_control": 1
+              "{clickhouseOperatorUser}/show_named_collections": 1
+              "{clickhouseOperatorUser}/show_named_collections_secrets": 1
+            profiles:
+            quotas:
+            settings:
+            files:
+
+      - version: ">= 23.5"
+        spec:
+          configuration:
+            users:
+            profiles:
+              clickhouse_operator/format_display_secrets_in_show_and_select: 1
+            quotas:
+            settings:
+              ##
+              ## this may be added on per-host basis into host's conf.d folder
+              ##
+              display_secrets_in_show_and_select: 1
+            files:
+
   #################################################
   ##
   ## Metrics collection
@@ -260,9 +315,9 @@ reconcile:
     #   3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward.
     # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage
 
-    # Max number of concurrent shard reconciles within one CHI in progress
+    # Max number of concurrent shard reconciles within one cluster in progress
     reconcileShardsThreadsNumber: 5
-    # Max percentage of concurrent shard reconciles within one CHI in progress
+    # Max percentage of concurrent shard reconciles within one cluster in progress
     reconcileShardsMaxConcurrencyPercent: 50
 
   # Reconcile StatefulSet scenario
@@ -306,6 +361,10 @@ reconcile:
       exclude: true
       queries: true
       include: false
+      replicas:
+        all: no
+        new: yes
+        delay: 10
 
 ################################################
 ##

deploy/builder/build-clickhouse-operator-install-yaml.sh

@@ -34,6 +34,8 @@ CH_PASSWORD_PLAIN="" \
 CH_CREDENTIALS_SECRET_NAME="clickhouse-operator" \
 CH_USERNAME_SECRET_PLAIN="clickhouse_operator" \
 CH_PASSWORD_SECRET_PLAIN="clickhouse_operator_password" \
+MANIFEST_PRINT_RBAC_NAMESPACED=yes \
+MANIFEST_PRINT_RBAC_CLUSTERED=yes \
 "${CUR_DIR}/cat-clickhouse-operator-install-yaml.sh" > "${MANIFEST_ROOT}/operator/clickhouse-operator-install-bundle.yaml"
 
 # Build templated installation .yaml manifest

deploy/builder/cat-clickhouse-operator-install-yaml.sh

@@ -156,8 +156,7 @@ if [[ "${MANIFEST_PRINT_CRD}" == "yes" ]]; then
         envsubst
 fi
 
-# Render RBAC section for ClusterRole
-if [[ "${MANIFEST_PRINT_RBAC_CLUSTERED}" == "yes" ]]; then
+if [[ "${MANIFEST_PRINT_RBAC_CLUSTERED}" == "yes" || "${MANIFEST_PRINT_RBAC_NAMESPACED}" == "yes" ]]; then
     # Render Account
     SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-02-section-rbac-01-service-account.yaml"
     ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
@@ -168,7 +167,10 @@ if [[ "${MANIFEST_PRINT_RBAC_CLUSTERED}" == "yes" ]]; then
         NAME="clickhouse-operator"                \
         OPERATOR_VERSION="${OPERATOR_VERSION}"    \
         envsubst
+fi
 
+# Render RBAC section for ClusterRole
+if [[ "${MANIFEST_PRINT_RBAC_CLUSTERED}" == "yes" ]]; then
     # Render Role
     SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-02-section-rbac-02-role.yaml"
     ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
@@ -186,16 +188,6 @@ fi
 
 # Render RBAC section for Role
 if [[ "${MANIFEST_PRINT_RBAC_NAMESPACED}" == "yes" ]]; then
-    # Render Account
-    SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-02-section-rbac-01-service-account.yaml"
-    ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
-    render_separator
-    cat "${TEMPLATES_DIR}/${SECTION_FILE_NAME}" | \
-        NAMESPACE="${OPERATOR_NAMESPACE}"         \
-        NAME="clickhouse-operator"                \
-        OPERATOR_VERSION="${OPERATOR_VERSION}"    \
-        envsubst
-
     # Render Role
     SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-02-section-rbac-02-role.yaml"
     ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
@@ -211,7 +203,6 @@ if [[ "${MANIFEST_PRINT_RBAC_NAMESPACED}" == "yes" ]]; then
         envsubst
 fi
 
-
 # Render header/beginning of ConfigMap yaml specification:
 # apiVersion: v1
 # kind: ConfigMap