@@ -2,10 +2,15 @@ package controller
22
33import (
44 "context"
5+ "errors"
6+ "fmt"
57 "strconv"
8+ "time"
69
710 kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1"
811 "github.com/enix/kube-image-keeper/internal/info"
12+ "github.com/obalunenko/getenv"
13+ "github.com/obalunenko/getenv/option"
914 "github.com/prometheus/client_golang/prometheus"
1015 "sigs.k8s.io/controller-runtime/pkg/client"
1116 "sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -16,6 +21,7 @@ import (
1621type kuikMetrics struct {
1722 collectors []prometheus.Collector
1823 monitoringTasks * prometheus.CounterVec
24+ // imageLastMonitorHistogram prometheus.Histogram
1925}
2026
2127var Metrics kuikMetrics
@@ -64,7 +70,7 @@ func (m *kuikMetrics) Register(elected <-chan struct{}, client client.Client) {
6470 )
6571 m .addCollector (m .monitoringTasks )
6672
67- m .addCollector (NewGenericCollector (
73+ m .addCollector (NewGenericCollectorFunc (
6874 prometheus.Opts {
6975 Namespace : info .MetricsNamespace ,
7076 Subsystem : subsystemRegistryMonitor ,
@@ -106,7 +112,7 @@ func (m *kuikMetrics) Register(elected <-chan struct{}, client client.Client) {
106112 }
107113 }))
108114
109- m .addCollector (NewGenericCollector (
115+ m .addCollector (NewGenericCollectorFunc (
110116 prometheus.Opts {
111117 Namespace : info .MetricsNamespace ,
112118 Subsystem : subsystemRegistryMonitor ,
@@ -131,6 +137,44 @@ func (m *kuikMetrics) Register(elected <-chan struct{}, client client.Client) {
131137 }
132138 }))
133139
140+ imageLastMonitorHistogramOpts := prometheus.Opts {
141+ Namespace : info .MetricsNamespace ,
142+ Subsystem : subsystemRegistryMonitor ,
143+ Name : "image_last_monitor_age_minutes" ,
144+ Help : "Histogram of image last monitor age in minutes" ,
145+ }
146+ imageLastMonitorHistogramLabels := []string {"registry" }
147+ imageLastMonitorHistogramBuckets , err := m .getImageLastMonitorAgeMinutesBuckets ()
148+ if err != nil {
149+ // TODO: handle error properly: return it to the caller
150+ panic (err )
151+ }
152+ // TODO: refactor NewGenericCollector to avoid duplicating Opts and labels usage
153+ m .addCollector (NewGenericCollector (imageLastMonitorHistogramOpts , prometheus .GaugeValue , imageLastMonitorHistogramLabels , func (ch chan <- prometheus.Metric ) {
154+ imageList := & kuikv1alpha1.ImageList {}
155+ if err := client .List (context .Background (), imageList ); err != nil {
156+ return
157+ }
158+
159+ histogram := prometheus .NewHistogramVec (prometheus.HistogramOpts {
160+ Namespace : imageLastMonitorHistogramOpts .Namespace ,
161+ Subsystem : imageLastMonitorHistogramOpts .Subsystem ,
162+ Name : imageLastMonitorHistogramOpts .Name ,
163+ Help : imageLastMonitorHistogramOpts .Help ,
164+ Buckets : imageLastMonitorHistogramBuckets ,
165+ }, imageLastMonitorHistogramLabels )
166+
167+ now := time .Now ()
168+ for _ , image := range imageList .Items {
169+ if image .Status .Upstream .LastMonitor .IsZero () {
170+ continue
171+ }
172+ histogram .WithLabelValues (image .Spec .Registry ).Observe (now .Sub (image .Status .Upstream .LastMonitor .Time ).Minutes ())
173+ }
174+
175+ histogram .Collect (ch )
176+ }))
177+
134178 metrics .Registry .MustRegister (m .collectors ... )
135179}
136180
@@ -147,27 +191,69 @@ func (m *kuikMetrics) InitMonitoringTaskRegistry(registry string) {
147191 }
148192}
149193
150- func (m * kuikMetrics ) MonitoringTaskCompleted (registry string , status kuikv1alpha1.ImageStatusUpstream , unused bool ) {
151- m .monitoringTasks .WithLabelValues (registry , status .ToString (), strconv .FormatBool (unused )).Inc ()
194+ func (m * kuikMetrics ) MonitoringTaskCompleted (registry string , status kuikv1alpha1.ImageStatus , unused bool ) {
195+ m .monitoringTasks .WithLabelValues (registry , status .Upstream .Status .ToString (), strconv .FormatBool (unused )).Inc ()
196+ }
197+
198+ func (m * kuikMetrics ) getImageLastMonitorAgeMinutesBuckets () ([]float64 , error ) {
199+ envPrefix := "KUIK_METRICS_IMAGE_LAST_MONITOR_AGE_MINUTES_BUCKETS_"
200+ switch bucketsType := getenv .EnvOrDefault (envPrefix + "TYPE" , "custom" ); bucketsType {
201+ case "exponential" :
202+ envPrefix += "EXPONENTIAL_"
203+ start , err := getenv.Env [float64 ](envPrefix + "START" )
204+ if err != nil {
205+ return nil , err
206+ }
207+ factor , err := getenv.Env [float64 ](envPrefix + "FACTOR" )
208+ if err != nil {
209+ return nil , err
210+ }
211+ count , err := getenv.Env [int ](envPrefix + "COUNT" )
212+ if err != nil {
213+ return nil , err
214+ }
215+ return prometheus .ExponentialBuckets (start , factor , count ), nil
216+ case "exponentialRange" :
217+ envPrefix += "EXPONENTIAL_RANGE_"
218+ minBucket , err := getenv.Env [float64 ](envPrefix + "MIN" )
219+ if err != nil {
220+ return nil , err
221+ }
222+ maxBucket , err := getenv.Env [float64 ](envPrefix + "MAX" )
223+ if err != nil {
224+ return nil , err
225+ }
226+ count , err := getenv.Env [int ](envPrefix + "COUNT" )
227+ if err != nil {
228+ return nil , err
229+ }
230+ return prometheus .ExponentialBucketsRange (minBucket , maxBucket , count ), nil
231+ case "custom" :
232+ buckets , err := getenv .Env [[]float64 ](envPrefix + "CUSTOM" , option .WithSeparator ("," ))
233+ if err != nil && errors .Is (err , getenv .ErrNotSet ) {
234+ return []float64 {2 , 10 }, nil
235+ }
236+ return buckets , err
237+ default :
238+ return nil , fmt .Errorf ("invalid %s: %s" , envPrefix + "TYPE" , bucketsType )
239+ }
152240}
153241
154242type GenericCollector struct {
155- desc * prometheus.Desc
156- valueType prometheus.ValueType
157- collectorCallback func ( collect func (value float64 , labels ... string ) )
243+ desc * prometheus.Desc
244+ valueType prometheus.ValueType
245+ callback func (ch chan <- prometheus. Metric )
158246}
159247
160248func (g * GenericCollector ) Describe (ch chan <- * prometheus.Desc ) {
161249 ch <- g .desc
162250}
163251
164252func (g * GenericCollector ) Collect (ch chan <- prometheus.Metric ) {
165- g .collectorCallback (func (value float64 , labels ... string ) {
166- ch <- prometheus .MustNewConstMetric (g .desc , g .valueType , value , labels ... )
167- })
253+ g .callback (ch )
168254}
169255
170- func NewGenericCollector (opts prometheus.Opts , valueType prometheus.ValueType , labels []string , collectorCallback func ( collect func ( value float64 , labels ... string ))) prometheus. Collector {
256+ func NewEmptyGenericCollector (opts prometheus.Opts , valueType prometheus.ValueType , labels []string ) * GenericCollector {
171257 desc := prometheus .NewDesc (
172258 prometheus .BuildFQName (opts .Namespace , opts .Subsystem , opts .Name ),
173259 opts .Help ,
@@ -176,8 +262,25 @@ func NewGenericCollector(opts prometheus.Opts, valueType prometheus.ValueType, l
176262 )
177263
178264 return & GenericCollector {
179- desc : desc ,
180- valueType : valueType ,
181- collectorCallback : collectorCallback ,
265+ desc : desc ,
266+ valueType : valueType ,
182267 }
183268}
269+
270+ func NewGenericCollector (opts prometheus.Opts , valueType prometheus.ValueType , labels []string , collectorCallback func (ch chan <- prometheus.Metric )) prometheus.Collector {
271+ collector := NewEmptyGenericCollector (opts , valueType , labels )
272+ collector .callback = collectorCallback
273+ return collector
274+ }
275+
276+ func NewGenericCollectorFunc (opts prometheus.Opts , valueType prometheus.ValueType , labels []string , callback func (collect func (value float64 , labels ... string ))) prometheus.Collector {
277+ collector := NewGenericCollector (opts , valueType , labels , nil ).(* GenericCollector )
278+
279+ collector .callback = func (ch chan <- prometheus.Metric ) {
280+ callback (func (value float64 , labels ... string ) {
281+ ch <- prometheus .MustNewConstMetric (collector .desc , collector .valueType , value , labels ... )
282+ })
283+ }
284+
285+ return collector
286+ }
0 commit comments