feat(metrics): setup prometheus exporter and export manager metrics

Author: plaffitt

Makefile

@@ -102,7 +102,7 @@ build: manifests generate fmt vet ## Build manager binary.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/main.go
+	go run ./cmd/main.go ${ARGS}
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.

cmd/main.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/alitto/pond/v2"
 	kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1"
+	"github.com/enix/kube-image-keeper/internal/controller"
 	corecontroller "github.com/enix/kube-image-keeper/internal/controller/core"
 	kuikcontroller "github.com/enix/kube-image-keeper/internal/controller/kuik"
 	// +kubebuilder:scaffold:imports
@@ -238,6 +239,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	controller.Metrics.Register(mgr.Elected())
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

helm/kube-image-keeper/templates/controller-deployment.yaml

@@ -54,6 +54,10 @@ spec:
             {{- end }}
             - name: no_proxy
               value: {{ join "," (prepend $noProxy (printf "%s-registry" (include "kube-image-keeper.fullname" .))) }}
+          ports:
+            - containerPort: 8080
+              name: metrics
+              protocol: TCP
           {{- with .Values.manager.readinessProbe }}
           readinessProbe:
             {{- toYaml . | nindent 12 }}

helm/kube-image-keeper/templates/controller-podmonitor.yaml

@@ -0,0 +1,23 @@
+{{- if .Values.manager.podMonitor.create }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "kube-image-keeper.fullname" . }}-manager
+  labels:
+    {{- include "kube-image-keeper.manager-labels" . | nindent 4 }}
+    {{- with .Values.manager.podMonitor.extraLabels }}
+    {{- . | toYaml | trim | nindent 4 }}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "kube-image-keeper.manager-selectorLabels" . | nindent 6 }}
+  podMetricsEndpoints:
+  - port: metrics
+    interval: {{ .Values.manager.podMonitor.scrapeInterval }}
+    scrapeTimeout: {{ .Values.manager.podMonitor.scrapeTimeout }}
+    {{- with .Values.manager.podMonitor.relabelings }}
+    relabelings:
+    {{- . | toYaml | nindent 4 }}
+    {{- end }}
+{{- end }}

helm/kube-image-keeper/values.yaml

@@ -60,6 +60,17 @@ manager:
       cpu: "1"
       # -- Memory limits for the manager pod
       memory: "512Mi"
+  podMonitor:
+    # -- Should a PodMonitor object be installed to scrape kuik manager metrics. For prometheus-operator (kube-prometheus) users.
+    create: false
+    # -- Target scrape interval set in the PodMonitor
+    scrapeInterval: 60s
+    # -- Target scrape timeout set in the PodMonitor
+    scrapeTimeout: 30s
+    # -- Additional labels to add to PodMonitor objects
+    extraLabels: {}
+    # -- Relabel config for the PodMonitor, see: https://coreos.com/operators/prometheus/docs/latest/api.html#relabelconfig
+    relabelings: []
 
 rbac:
   # -- Create the ClusterRole and ClusterRoleBinding. If false, need to associate permissions with serviceAccount outside this Helm chart.

internal/controller/collector.go

@@ -0,0 +1,52 @@
+package controller
+
+import (
+	"github.com/enix/kube-image-keeper/internal/info"
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+type kuikMetrics struct {
+	collectors []prometheus.Collector
+}
+
+var Metrics kuikMetrics
+
+func (m *kuikMetrics) Register(elected <-chan struct{}) {
+	const subsystem = "manager"
+
+	m.addCollector(info.NewInfoCollector(subsystem))
+
+	m.addCollector(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: info.MetricsNamespace,
+		Subsystem: subsystem,
+		Name:      "is_leader",
+		Help:      "Whether or not this replica is a leader. 1 if it is, 0 otherwise.",
+	}, func() float64 {
+		select {
+		case <-elected:
+			return 1
+		default:
+			return 0
+		}
+	}))
+
+	m.addCollector(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: info.MetricsNamespace,
+		Subsystem: subsystem,
+		Name:      "up",
+		Help:      "Whether or not this replica is healthy.",
+	}, func() float64 {
+		if err := healthz.Ping(nil); err != nil {
+			return 0
+		}
+		return 1
+	}))
+
+	metrics.Registry.MustRegister(m.collectors...)
+}
+
+func (m *kuikMetrics) addCollector(collector prometheus.Collector) {
+	m.collectors = append(m.collectors, collector)
+}

internal/controller/kuik/registrymonitor_controller.go

@@ -91,16 +91,18 @@ func (r *RegistryMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		logImage := logf.Log.WithValues("controller", "imagemonitor", "image", klog.KObj(&image), "reference", image.Reference()).V(1)
 		logImage.Info("queuing image for monitoring")
 
-		monitorPool.Submit(func() {
+		monitorPool.SubmitErr(func() error {
 			logImage.Info("monitoring image")
 			if err := r.monitorAnImage(logf.IntoContext(context.Background(), logImage), &image); err != nil {
 				logImage.Info("failed to monitor image", "error", err.Error())
+				return err
 			}
 			logImage.Info("image monitored with success")
+			return nil
 		})
 	}
 
-	log.Info("queued images for monitoring with success")
+	log.Info("queued images for monitoring with success", "completed", monitorPool.CompletedTasks(), "failed", monitorPool.FailedTasks())
 
 	return ctrl.Result{RequeueAfter: registryMonitor.Spec.Interval.Duration / time.Duration(registryMonitor.Spec.MaxPerInterval)}, nil
 }

internal/info/info.go

@@ -0,0 +1,48 @@
+package info
+
+import (
+	"runtime"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const MetricsNamespace = "kube_image_keeper"
+
+// Version and build informations set at link time
+var (
+	Version       = "0.0.0"
+	Revision      = ""
+	BuildDateTime = ""
+)
+
+type InfoCollector struct {
+	metric prometheus.Metric
+}
+
+// Describe implements Collector.
+func (i *InfoCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- i.metric.Desc()
+}
+
+// Collect implements Collector.
+func (i *InfoCollector) Collect(ch chan<- prometheus.Metric) {
+	ch <- i.metric
+}
+
+func NewInfoCollector(subsystem string) prometheus.Collector {
+	infoMetric := prometheus.BuildFQName(MetricsNamespace, subsystem, "build_info")
+	infoHelp := "A metric with a constant '1' value labeled with version, revision, build date, Go version, Go OS, and Go architecture"
+	infoConstLabels := prometheus.Labels{
+		"version":   Version,
+		"revision":  Revision,
+		"built":     BuildDateTime,
+		"goversion": runtime.Version(),
+		"goos":      runtime.GOOS,
+		"goarch":    runtime.GOARCH,
+	}
+	infoDesc := prometheus.NewDesc(infoMetric, infoHelp, nil, infoConstLabels)
+
+	return &InfoCollector{
+		metric: prometheus.MustNewConstMetric(infoDesc, prometheus.GaugeValue, float64(1)),
+	}
+}