feat: Improve error handling in prometheus exporter (#7363)

fix #7066

---------

Co-authored-by: Tyler Yahn <[email protected]>

Author: tongoss

CHANGELOG.md

@@ -42,6 +42,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - `WithInstrumentationAttributes` in `go.opentelemetry.io/otel/meter` synchronously de-duplicates the passed attributes instead of delegating it to the returned `MeterOption`. (#7266)
 - `WithInstrumentationAttributes` in `go.opentelemetry.io/otel/log` synchronously de-duplicates the passed attributes instead of delegating it to the returned `LoggerOption`. (#7266)
 - `Distinct` in `go.opentelemetry.io/otel/attribute` is no longer guaranteed to uniquely identify an attribute set. Collisions between `Distinct` values for different Sets are possible with extremely high cardinality (billions of series per instrument), but are highly unlikely. (#7175)
+- Improve error handling for dropped data during translation by using `prometheus.NewInvalidMetric` in `go.opentelemetry.io/otel/exporters/prometheus`.
+  **Breaking Change:** Previously, these cases were only logged and scrapes succeeded.
+  Now, when translation would drop data (e.g., invalid label/value), the exporter emits a `NewInvalidMetric`, and Prometheus scrapes **fail with HTTP 500** by default.
+  To preserve the prior behavior (scrapes succeed while errors are logged), configure your Prometheus HTTP handler with: `promhttp.HandlerOpts{ ErrorHandling: promhttp.ContinueOnError }`. (#7363)
 - The default `TranslationStrategy` in `go.opentelemetry.io/exporters/prometheus` is changed from `otlptranslator.NoUTF8EscapingWithSuffixes` to `otlptranslator.UnderscoreEscapingWithSuffixes`. (#7421)
 - The `ErrorType` function in `go.opentelemetry.io/otel/semconv/v1.37.0` now handles custom error types.
   If an error implements an `ErrorType() string` method, the return value of that method will be used as the error type. (#7442)

exporters/prometheus/errors.go

@@ -0,0 +1,13 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package prometheus // import "go.opentelemetry.io/otel/exporters/prometheus"
+
+import "errors"
+
+// Sentinel errors for consistent error checks in tests.
+var (
+	errInvalidMetricType = errors.New("invalid metric type")
+	errInvalidMetric     = errors.New("invalid metric")
+	errEHScaleBelowMin   = errors.New("exponential histogram scale below minimum supported")
+)

exporters/prometheus/exporter.go

@@ -241,7 +241,7 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 
 			attrKeys, attrVals, e := getAttrs(scopeMetrics.Scope.Attributes, c.labelNamer)
 			if e != nil {
-				otel.Handle(e)
+				reportError(ch, nil, e)
 				err = errors.Join(err, fmt.Errorf("failed to getAttrs for ScopeMetrics %d: %w", j, e))
 				continue
 			}
@@ -258,19 +258,19 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 		for k, m := range scopeMetrics.Metrics {
 			typ := c.metricType(m)
 			if typ == nil {
+				reportError(ch, nil, errInvalidMetricType)
 				continue
 			}
 			name, e := c.getName(m)
 			if e != nil {
-				// TODO(#7066): Handle this error better. It's not clear this can be
-				// reached, bad metric names should / will be caught at creation time.
-				otel.Handle(e)
+				reportError(ch, nil, e)
 				err = errors.Join(err, fmt.Errorf("failed to getAttrs for ScopeMetrics %d, Metrics %d: %w", j, k, e))
 				continue
 			}
 
 			drop, help := c.validateMetrics(name, m.Description, typ)
 			if drop {
+				reportError(ch, nil, errInvalidMetric)
 				continue
 			}
 
@@ -373,7 +373,7 @@ func addExponentialHistogramMetric[N int64 | float64](
 	for j, dp := range histogram.DataPoints {
 		keys, values, e := getAttrs(dp.Attributes, labelNamer)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, nil, e)
 			err = errors.Join(err, fmt.Errorf("failed to getAttrs for histogram.DataPoints %d: %w", j, e))
 			continue
 		}
@@ -386,11 +386,11 @@ func addExponentialHistogramMetric[N int64 | float64](
 		scale := dp.Scale
 		if scale < -4 {
 			// Reject scales below -4 as they cannot be represented in Prometheus
-			e := fmt.Errorf(
-				"exponential histogram scale %d is below minimum supported scale -4, skipping data point",
-				scale,
+			reportError(
+				ch,
+				desc,
+				fmt.Errorf("%w: %d (min -4)", errEHScaleBelowMin, scale),
 			)
-			otel.Handle(e)
 			err = errors.Join(err, e)
 			continue
 		}
@@ -440,7 +440,7 @@ func addExponentialHistogramMetric[N int64 | float64](
 			dp.StartTime,
 			values...)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, desc, e)
 			err = errors.Join(
 				err,
 				fmt.Errorf("failed to NewConstNativeHistogram for histogram.DataPoints %d: %w", j, e),
@@ -474,7 +474,7 @@ func addHistogramMetric[N int64 | float64](
 	for j, dp := range histogram.DataPoints {
 		keys, values, e := getAttrs(dp.Attributes, labelNamer)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, nil, e)
 			err = errors.Join(err, fmt.Errorf("failed to getAttrs for histogram.DataPoints %d: %w", j, e))
 			continue
 		}
@@ -491,7 +491,7 @@ func addHistogramMetric[N int64 | float64](
 		}
 		m, e := prometheus.NewConstHistogram(desc, dp.Count, float64(dp.Sum), buckets, values...)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, desc, e)
 			err = errors.Join(err, fmt.Errorf("failed to NewConstMetric for histogram.DataPoints %d: %w", j, e))
 			continue
 		}
@@ -527,7 +527,7 @@ func addSumMetric[N int64 | float64](
 	for i, dp := range sum.DataPoints {
 		keys, values, e := getAttrs(dp.Attributes, labelNamer)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, nil, e)
 			err = errors.Join(err, fmt.Errorf("failed to getAttrs for sum.DataPoints %d: %w", i, e))
 			continue
 		}
@@ -537,7 +537,7 @@ func addSumMetric[N int64 | float64](
 		desc := prometheus.NewDesc(name, m.Description, keys, nil)
 		m, e := prometheus.NewConstMetric(desc, valueType, float64(dp.Value), values...)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, desc, e)
 			err = errors.Join(err, fmt.Errorf("failed to NewConstMetric for sum.DataPoints %d: %w", i, e))
 			continue
 		}
@@ -572,7 +572,7 @@ func addGaugeMetric[N int64 | float64](
 	for i, dp := range gauge.DataPoints {
 		keys, values, e := getAttrs(dp.Attributes, labelNamer)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, nil, e)
 			err = errors.Join(err, fmt.Errorf("failed to getAttrs for gauge.DataPoints %d: %w", i, e))
 			continue
 		}
@@ -582,7 +582,7 @@ func addGaugeMetric[N int64 | float64](
 		desc := prometheus.NewDesc(name, m.Description, keys, nil)
 		m, e := prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(dp.Value), values...)
 		if e != nil {
-			otel.Handle(e)
+			reportError(ch, desc, e)
 			err = errors.Join(err, fmt.Errorf("failed to NewConstMetric for gauge.DataPoints %d: %w", i, e))
 			continue
 		}
@@ -803,3 +803,10 @@ func attributesToLabels(attrs []attribute.KeyValue, labelNamer otlptranslator.La
 	}
 	return labels, nil
 }
+
+func reportError(ch chan<- prometheus.Metric, desc *prometheus.Desc, err error) {
+	if desc == nil {
+		desc = prometheus.NewInvalidDesc(err)
+	}
+	ch <- prometheus.NewInvalidMetric(desc, err)
+}