Reintroduce thanos_objstore_bucket_operation_failures_total alert

Metric was removed on PR 2002 by mistake

Signed-off-by: dmaiocchi <dmaiocchi@suse.com>

Author: MalloZup

examples/alerts/alerts.md

@@ -314,6 +314,17 @@ rules:
   for: 5m
   labels:
     severity: critical
+- alert: ThanosSidecarBucketOperationsFailed
+  annotations:
+    description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations
+      are failing
+    runbook_url: https://github.com/thanos-io/thanos/tree/master/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed
+    summary: Thanos Sidecar cannot connect to Prometheus
+  expr: |
+    rate(thanos_objstore_bucket_operation_failures_total{name="prometheus"}[5m]) > 0
+  for: 5m
+  labels:
+    severity: critical
 - alert: ThanosSidecarUnhealthy
   annotations:
     description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{

examples/alerts/alerts.yaml

@@ -295,6 +295,17 @@ groups:
     for: 5m
     labels:
       severity: critical
+  - alert: ThanosSidecarBucketOperationsFailed
+    annotations:
+      description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations
+        are failing
+      runbook_url: https://github.com/thanos-io/thanos/tree/master/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed
+      summary: Thanos Sidecar cannot connect to Prometheus
+    expr: |
+      rate(thanos_objstore_bucket_operation_failures_total{name="prometheus"}[5m]) > 0
+    for: 5m
+    labels:
+      severity: critical
   - alert: ThanosSidecarUnhealthy
     annotations:
       description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for

foo

@@ -0,0 +1 @@
+fdfd

mixin/alerts/sidecar.libsonnet

@@ -22,6 +22,20 @@
               severity: 'critical',
             },
           },
+          {
+            alert: 'ThanosSidecarBucketOperationsFailed',
+            annotations: {
+              description: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations are failing',
+              summary: 'Thanos Sidecar cannot connect to Prometheus',
+            },
+            expr: |||
+              rate(thanos_objstore_bucket_operation_failures_total{name="prometheus"}[5m]) > 0
+            ||| % thanos.sidecar,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+          },
           {
             alert: 'ThanosSidecarUnhealthy',
             annotations: {

pkg/rules/rules_test.go

@@ -82,7 +82,7 @@ func testRulesAgainstExamples(t *testing.T, dir string, server rulespb.RulesServ
 		{
 			Name:                    "thanos-sidecar",
 			File:                    filepath.Join(dir, "alerts.yaml"),
-			Rules:                   []*rulespb.Rule{someAlert, someAlert},
+			Rules:                   []*rulespb.Rule{someAlert, someAlert, someAlert},
 			Interval:                60,
 			PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT,
 		},