[prometheus] Fix collection output buffer management when its resized (#5676)

Co-authored-by: Reiley Yang <reyang@microsoft.com>
Co-authored-by: Mikel Blanchard <mblanchard@macrosssoftware.com>
Co-authored-by: Vishwesh Bankwar <vishweshbankwar@users.noreply.github.com>

Author: 4 people

src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+* Fixed a bug which lead to empty responses when the internal buffer is resized
+  processing a collection request
+  ([#5676](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5676))
+
 ## 1.9.0-beta.1
 
 Released 2024-Jun-14

src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+* Fixed a bug which lead to empty responses when the internal buffer is resized
+  processing a collection request
+  ([#5676](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5676))
+
 ## 1.9.0-beta.1
 
 Released 2024-Jun-14

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs

@@ -198,13 +198,13 @@ private bool ExecuteCollect(bool openMetricsRequested)
     private ExportResult OnCollect(Batch<Metric> metrics)
     {
         var cursor = 0;
-        var buffer = this.exporter.OpenMetricsRequested ? this.openMetricsBuffer : this.plainTextBuffer;
+        ref byte[] buffer = ref (this.exporter.OpenMetricsRequested ? ref this.openMetricsBuffer : ref this.plainTextBuffer);
 
         try
         {
             if (this.exporter.OpenMetricsRequested)
             {
-                cursor = this.WriteTargetInfo();
+                cursor = this.WriteTargetInfo(ref buffer);
 
                 this.scopes.Clear();
 
@@ -291,11 +291,11 @@ private ExportResult OnCollect(Batch<Metric> metrics)
 
             if (this.exporter.OpenMetricsRequested)
             {
-                this.previousOpenMetricsDataView = new ArraySegment<byte>(this.openMetricsBuffer, 0, cursor);
+                this.previousOpenMetricsDataView = new ArraySegment<byte>(buffer, 0, cursor);
             }
             else
             {
-                this.previousPlainTextDataView = new ArraySegment<byte>(this.plainTextBuffer, 0, cursor);
+                this.previousPlainTextDataView = new ArraySegment<byte>(buffer, 0, cursor);
             }
 
             return ExportResult.Success;
@@ -315,21 +315,21 @@ private ExportResult OnCollect(Batch<Metric> metrics)
         }
     }
 
-    private int WriteTargetInfo()
+    private int WriteTargetInfo(ref byte[] buffer)
     {
         if (this.targetInfoBufferLength < 0)
         {
             while (true)
             {
                 try
                 {
-                    this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(this.openMetricsBuffer, 0, this.exporter.Resource);
+                    this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(buffer, 0, this.exporter.Resource);
 
                     break;
                 }
                 catch (IndexOutOfRangeException)
                 {
-                    if (!this.IncreaseBufferSize(ref this.openMetricsBuffer))
+                    if (!this.IncreaseBufferSize(ref buffer))
                     {
                         throw;
                     }

test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs

@@ -288,6 +288,30 @@ public async Task PrometheusExporterMiddlewareIntegration_CanServeOpenMetricsAnd
         await host.StopAsync();
     }
 
+    [Fact]
+    public async Task PrometheusExporterMiddlewareIntegration_TestBufferSizeIncrease_With_LotOfMetrics()
+    {
+        using var host = await StartTestHostAsync(
+            app => app.UseOpenTelemetryPrometheusScrapingEndpoint());
+
+        using var meter = new Meter(MeterName, MeterVersion);
+
+        for (var x = 0; x < 1000; x++)
+        {
+            var counter = meter.CreateCounter<double>("counter_double_" + x, unit: "By");
+            counter.Add(1);
+        }
+
+        using var client = host.GetTestClient();
+
+        using var response = await client.GetAsync("/metrics");
+        var text = await response.Content.ReadAsStringAsync();
+
+        Assert.NotEmpty(text);
+
+        await host.StopAsync();
+    }
+
     private static async Task RunPrometheusExporterMiddlewareIntegrationTest(
         string path,
         Action<IApplicationBuilder> configure,

test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs

@@ -144,6 +144,45 @@ public void PrometheusHttpListenerThrowsOnStart()
         listener?.Dispose();
     }
 
+    [Theory]
+    [InlineData("application/openmetrics-text")]
+    [InlineData("")]
+    public async Task PrometheusExporterHttpServerIntegration_TestBufferSizeIncrease_With_LargePayload(string acceptHeader)
+    {
+        using var meter = new Meter(MeterName, MeterVersion);
+
+        var attributes = new List<KeyValuePair<string, object>>();
+        var oneKb = new string('A', 1024);
+        for (var x = 0; x < 8500; x++)
+        {
+            attributes.Add(new KeyValuePair<string, object>(x.ToString(), oneKb));
+        }
+
+        var provider = BuildMeterProvider(meter, attributes, out var address);
+
+        for (var x = 0; x < 1000; x++)
+        {
+            var counter = meter.CreateCounter<double>("counter_double_" + x, unit: "By");
+            counter.Add(1);
+        }
+
+        using HttpClient client = new HttpClient();
+
+        if (!string.IsNullOrEmpty(acceptHeader))
+        {
+            client.DefaultRequestHeaders.Add("Accept", acceptHeader);
+        }
+
+        using var response = await client.GetAsync($"{address}metrics");
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        var content = await response.Content.ReadAsStringAsync();
+        Assert.Contains("counter_double_999", content);
+        Assert.DoesNotContain('\0', content);
+
+        provider.Dispose();
+    }
+
     private static void TestPrometheusHttpListenerUriPrefixOptions(string[] uriPrefixes)
     {
         using var exporter = new PrometheusExporter(new());
@@ -155,31 +194,27 @@ private static void TestPrometheusHttpListenerUriPrefixOptions(string[] uriPrefi
             });
     }
 
-    private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetrics = false, string acceptHeader = "application/openmetrics-text")
+    private static MeterProvider BuildMeterProvider(Meter meter, IEnumerable<KeyValuePair<string, object>> attributes, out string address)
     {
-        var requestOpenMetrics = acceptHeader.StartsWith("application/openmetrics-text");
-
         Random random = new Random();
         int retryAttempts = 5;
         int port = 0;
-        string address = null;
-
+        string generatedAddress = null;
         MeterProvider provider = null;
-        using var meter = new Meter(MeterName, MeterVersion);
 
         while (retryAttempts-- != 0)
         {
             port = random.Next(2000, 5000);
-            address = $"http://localhost:{port}/";
+            generatedAddress = $"http://localhost:{port}/";
 
             try
             {
                 provider = Sdk.CreateMeterProviderBuilder()
                     .AddMeter(meter.Name)
-                    .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1"))
+                    .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1").AddAttributes(attributes))
                     .AddPrometheusHttpListener(options =>
                     {
-                        options.UriPrefixes = new string[] { address };
+                        options.UriPrefixes = new string[] { generatedAddress };
                     })
                     .Build();
 
@@ -191,11 +226,24 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri
             }
         }
 
+        address = generatedAddress;
+
         if (provider == null)
         {
             throw new InvalidOperationException("HttpListener could not be started");
         }
 
+        return provider;
+    }
+
+    private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetrics = false, string acceptHeader = "application/openmetrics-text")
+    {
+        var requestOpenMetrics = acceptHeader.StartsWith("application/openmetrics-text");
+
+        using var meter = new Meter(MeterName, MeterVersion);
+
+        var provider = BuildMeterProvider(meter, [], out var address);
+
         var tags = new KeyValuePair<string, object>[]
         {
             new KeyValuePair<string, object>("key1", "value1"),