chore: add built-in Prometheus support

CHANGELOG.md

@@ -27,6 +27,8 @@ All notable changes to this project will be documented in this file.
   - The `runAsUser` and `runAsGroup` fields will not be set anymore by the operator
   - The defaults from the docker images itself will now apply, which will be different from 1000/0 going forward
   - This is marked as breaking because tools and policies might exist, which require these fields to be set
+- BREAKING: Replace JMX Exporter with built in Prometheus support. Hence, the metrics provided by the `/metrics` endpoint are named
+    differently now ([#955]).
 
 ### Fixed
 
@@ -45,6 +47,7 @@ All notable changes to this project will be documented in this file.
 [#942]: https://github.com/stackabletech/zookeeper-operator/pull/942
 [#946]: https://github.com/stackabletech/zookeeper-operator/pull/946
 [#950]: https://github.com/stackabletech/zookeeper-operator/pull/950
+[#955]: https://github.com/stackabletech/zookeeper-operator/pull/955
 
 ## [25.3.0] - 2025-03-21
 

docs/modules/zookeeper/pages/usage_guide/monitoring.adoc

@@ -2,4 +2,23 @@
 :description: The managed ZooKeeper instances are automatically configured to export Prometheus metrics.
 
 The managed ZooKeeper instances are automatically configured to export Prometheus metrics.
-See xref:operators:monitoring.adoc[] for more details.
+See xref:operators:monitoring.adoc[window=_blank] for more details.
+
+Depending on the SDP version, different ZooKeeper monitoring systems are used to produce metrics.
+Previously, JMX in combination with JMX Exporter was used. Starting with SDP 25.7 the New Metrics System is utilized.
+The naming of the metrics differs between the two systems.
+
+== ZooKeeper New Metrics System
+
+Since SDP 25.7 ZooKeeper is configured to provide metrics by utilizing the New Metrics System. More on the New Metrics System in
+the https://zookeeper.apache.org/doc/current/zookeeperMonitor.html[ZooKeeper Monitor Guide,window=_blank].
+
+The configuration is located in the `zoo.cfg`:
+
+[source,properties]
+----
+metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
+metricsProvider.httpPort=9505
+----
+
+The metrics can be accessed by calling the `/metrics` endpoint on the specified port.

rust/operator-binary/src/config/jvm.rs

@@ -6,7 +6,7 @@ use stackable_operator::{
 
 use crate::crd::{
     JVM_SECURITY_PROPERTIES_FILE, LOG4J_CONFIG_FILE, LOGBACK_CONFIG_FILE, LoggingFramework,
-    METRICS_PORT, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR,
+    STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR,
     v1alpha1::{ZookeeperCluster, ZookeeperConfig, ZookeeperConfigFragment},
 };
 
@@ -36,9 +36,6 @@ fn construct_jvm_args(
 
     let jvm_args = vec![
         format!("-Djava.security.properties={STACKABLE_CONFIG_DIR}/{JVM_SECURITY_PROPERTIES_FILE}"),
-        format!(
-            "-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/server.yaml"
-        ),
         match logging_framework {
             LoggingFramework::LOG4J => {
                 format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}")
@@ -123,7 +120,6 @@ mod tests {
         assert_eq!(
             non_heap_jvm_args,
             "-Djava.security.properties=/stackable/config/security.properties \
-            -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=9505:/stackable/jmx/server.yaml \
             -Dlogback.configurationFile=/stackable/log_config/logback.xml"
         );
         assert_eq!(zk_server_heap_env, "409");
@@ -168,7 +164,6 @@ mod tests {
         assert_eq!(
             non_heap_jvm_args,
             "-Djava.security.properties=/stackable/config/security.properties \
-            -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=9505:/stackable/jmx/server.yaml \
             -Dlogback.configurationFile=/stackable/log_config/logback.xml \
             -Dhttps.proxyHost=proxy.my.corp \
             -Djava.net.preferIPv4Stack=true \

rust/operator-binary/src/crd/mod.rs

@@ -47,7 +47,9 @@ pub const OPERATOR_NAME: &str = "zookeeper.stackable.tech";
 pub const ZOOKEEPER_PROPERTIES_FILE: &str = "zoo.cfg";
 pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties";
 
-pub const METRICS_PORT: u16 = 9505;
+pub const METRICS_PROVIDER_CLASS_NAME: &str =
+    "org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider";
+pub const METRICS_PROVIDER_HTTP_PORT: u16 = 9505;
 
 pub const STACKABLE_DATA_DIR: &str = "/stackable/data";
 pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config";
@@ -371,6 +373,8 @@ impl v1alpha1::ZookeeperConfig {
     pub const DATA_DIR: &'static str = "dataDir";
     const DEFAULT_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1);
     pub const INIT_LIMIT: &'static str = "initLimit";
+    const METRICS_PROVIDER_CLASS_NAME: &'static str = "metricsProvider.className";
+    const METRICS_PROVIDER_HTTP_PORT: &'static str = "metricsProvider.httpPort";
     pub const MYID_OFFSET: &'static str = "MYID_OFFSET";
     pub const SYNC_LIMIT: &'static str = "syncLimit";
     pub const TICK_TIME: &'static str = "tickTime";
@@ -468,6 +472,14 @@ impl Configuration for v1alpha1::ZookeeperConfigFragment {
                 v1alpha1::ZookeeperConfig::DATA_DIR.to_string(),
                 Some(STACKABLE_DATA_DIR.to_string()),
             );
+            result.insert(
+                v1alpha1::ZookeeperConfig::METRICS_PROVIDER_CLASS_NAME.to_string(),
+                Some(METRICS_PROVIDER_CLASS_NAME.to_string()),
+            );
+            result.insert(
+                v1alpha1::ZookeeperConfig::METRICS_PROVIDER_HTTP_PORT.to_string(),
+                Some(METRICS_PROVIDER_HTTP_PORT.to_string()),
+            );
         }
 
         Ok(result)

tests/templates/kuttl/smoke/test_zookeeper.py

@@ -3,6 +3,7 @@
 import requests
 import time
 import sys
+
 sys.tracebacklimit = 0
 
 
@@ -37,36 +38,67 @@ def check_ruok(hosts):
         url = host + ":8080/commands/" + cmd_ruok
         response = try_get(url).json()
 
-        if "command" in response and response["command"] == cmd_ruok \
-                and "error" in response and response["error"] is None:
+        if (
+            "command" in response
+            and response["command"] == cmd_ruok
+            and "error" in response
+            and response["error"] is None
+        ):
             continue
         else:
-            print("Error[" + cmd_ruok + "] for [" + url + "]: received " + str(
-                response) + " - expected {'command': 'ruok', 'error': None} ")
+            print(
+                "Error["
+                + cmd_ruok
+                + "] for ["
+                + url
+                + "]: received "
+                + str(response)
+                + " - expected {'command': 'ruok', 'error': None} "
+            )
             exit(-1)
 
 
 def check_monitoring(hosts):
     for host in hosts:
-        url = host + ":9505"
+        url = host + ":9505/metrics"
         response = try_get(url)
 
         if response.ok:
+            # arbitrary metric was chosen to test if metrics are present in the response
+            if "quorum_size" in response.text:
+                continue
+            else:
+                print("Error for [" + url + "]: missing metrics")
+                exit(-1)
             continue
         else:
             print("Error for [" + url + "]: could not access monitoring")
             exit(-1)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     all_args = argparse.ArgumentParser(description="Test ZooKeeper.")
-    all_args.add_argument("-n", "--namespace", help="The namespace to run in", required=True)
+    all_args.add_argument(
+        "-n", "--namespace", help="The namespace to run in", required=True
+    )
     args = vars(all_args.parse_args())
     namespace = args["namespace"]
 
-    host_primary_0 = "http://test-zk-server-primary-0.test-zk-server-primary." + namespace + ".svc.cluster.local"
-    host_primary_1 = "http://test-zk-server-primary-1.test-zk-server-primary." + namespace + ".svc.cluster.local"
-    host_secondary = "http://test-zk-server-secondary-0.test-zk-server-secondary." + namespace + ".svc.cluster.local"
+    host_primary_0 = (
+        "http://test-zk-server-primary-0.test-zk-server-primary."
+        + namespace
+        + ".svc.cluster.local"
+    )
+    host_primary_1 = (
+        "http://test-zk-server-primary-1.test-zk-server-primary."
+        + namespace
+        + ".svc.cluster.local"
+    )
+    host_secondary = (
+        "http://test-zk-server-secondary-0.test-zk-server-secondary."
+        + namespace
+        + ".svc.cluster.local"
+    )
 
     hosts = [host_primary_0, host_primary_1, host_secondary]