feat!: update all bundled metrics configs to export Prometheus metrics

packaging/docker/Dockerfile

@@ -4,8 +4,8 @@ COPY observe-agent /
 # This shouldn't be necessary but sometimes we end up with execution bit not set.
 # ref: https://github.com/open-telemetry/opentelemetry-collector/issues/1317
 RUN chmod 755 /observe-agent
-# The base executable includes symlinks to libraries. In order to break those, 
-# we tar with the h flag to dereference symlinks. 
+# The base executable includes symlinks to libraries. In order to break those,
+# we tar with the h flag to dereference symlinks.
 RUN tar czhf observe-agent.tar.gz /observe-agent $(ldd /observe-agent | grep -oP "\/.*? ")
 # extract package to /output so it can be taken as base for scratch image
 # we do not copy archive into scratch image, as it doesn't have tar executable

packaging/docker/compose-linux-host.yaml

@@ -25,7 +25,7 @@ services:
         source: /var/log
         target: /hostfs/var/log
         read_only: true
-      # Symlinks dont work by default on docker containers so we need to add each of the 
+      # Symlinks dont work by default on docker containers so we need to add each of the
       # directories containing symlink targets individually for docker to be able to follow them
       - type: bind
         source: /var/log/pods

packaging/docker/observe-agent/connections/host_monitoring/host.yaml

@@ -9,4 +9,4 @@ service:
     metrics/agent-filestats:
        receivers: [filestats/agent]
        processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
+       exporters: [prometheusremotewrite/observe]

packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml

@@ -37,4 +37,4 @@ service:
     metrics/host_monitoring_host:
       receivers: [hostmetrics/host-monitoring-host]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml

@@ -10,15 +10,15 @@ receivers:
           process.cpu.utilization:
             enabled: true
           process.disk.operations:
-            enabled: true      
+            enabled: true
           process.memory.utilization:
-            enabled: true      
+            enabled: true
           process.open_file_descriptors:
-            enabled: true      
+            enabled: true
           process.paging.faults:
-            enabled: true      
+            enabled: true
           process.signals_pending:
-            enabled: true      
+            enabled: true
           process.threads:
             enabled: true
         mute_process_name_error: true
@@ -31,4 +31,4 @@ service:
     metrics/host_monitoring_process:
       receivers: [hostmetrics/host-monitoring-process]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml

@@ -27,8 +27,8 @@ service:
   pipelines:
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
-      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
     logs/agent-journald:
       receivers: [journald/agent]

packaging/docker/observe-agent/otel-collector.yaml

@@ -22,7 +22,7 @@ processors:
   transform/truncate:
     log_statements:
       - context: log
-        statements: 
+        statements:
           - truncate_all(attributes, 4095)
           - truncate_all(resource.attributes, 4095)
     trace_statements:
@@ -35,9 +35,11 @@ processors:
     check_interval: 1s
     limit_percentage: 20
     spike_limit_percentage: 5
-  
+
   batch:
-
+
+  deltatocumulative:
+
   resourcedetection:
     detectors: [env, system]
     system:
@@ -65,7 +67,7 @@ processors:
           enabled: true
         os.description:
           enabled: true
-  
+
   resourcedetection/cloud:
     detectors:
     {{- if .CloudResourceDetectors }}
@@ -74,7 +76,7 @@ processors:
     {{- end }}
     timeout: 2s
     override: false
-    
+
   filter/count:
     error_mode: ignore
     metrics:
@@ -93,16 +95,24 @@ exporters:
       enabled: true
     compression: zstd
 
+  prometheusremotewrite/observe:
+    endpoint: ${env:OBSERVE_PROMETHEUS_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+    resource_to_telemetry_conversion:
+      enabled: true  # Convert resource attributes to metric labels
+    send_metadata: true
+
   debug:
 
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
-      processors: [resourcedetection, resourcedetection/cloud]
-      exporters: [otlphttp/observe]
+      processors: [resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
-    logs/forward: 
+    logs/forward:
       receivers: [otlp]
       processors: [resourcedetection, resourcedetection/cloud]
       exporters: [otlphttp/observe, count]

packaging/linux/etc/observe-agent/connections/host_monitoring/host.yaml

@@ -9,4 +9,4 @@ service:
     metrics/agent-filestats:
        receivers: [filestats/agent]
        processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
+       exporters: [prometheusremotewrite/observe]

packaging/linux/etc/observe-agent/connections/host_monitoring/host_metrics.yaml

@@ -36,4 +36,4 @@ service:
     metrics/host_monitoring_host:
       receivers: [hostmetrics/host-monitoring-host]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/linux/etc/observe-agent/connections/host_monitoring/process_metrics.yaml

@@ -30,4 +30,4 @@ service:
     metrics/host_monitoring_process:
       receivers: [hostmetrics/host-monitoring-process]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/linux/etc/observe-agent/connections/self_monitoring/logs_and_metrics.yaml

@@ -27,8 +27,8 @@ service:
   pipelines:
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
-      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
     logs/agent-journald:
       receivers: [journald/agent]

packaging/linux/etc/observe-agent/otel-collector.yaml

@@ -38,6 +38,8 @@ processors:
 
   batch:
 
+  deltatocumulative:
+
   resourcedetection:
     detectors: [env, system]
     system:
@@ -93,14 +95,22 @@ exporters:
       enabled: true
     compression: zstd
 
+  prometheusremotewrite/observe:
+    endpoint: ${env:OBSERVE_PROMETHEUS_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+    resource_to_telemetry_conversion:
+      enabled: true  # Convert resource attributes to metric labels
+    send_metadata: true
+
   debug:
 
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
-      processors: [resourcedetection, resourcedetection/cloud]
-      exporters: [otlphttp/observe]
+      processors: [resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
     logs/forward:
       receivers: [otlp]

packaging/macos/config/otel-collector.yaml

@@ -38,6 +38,8 @@ processors:
 
   batch:
 
+  deltatocumulative:
+
   resourcedetection:
     detectors: [env, system]
     system:
@@ -93,14 +95,22 @@ exporters:
       enabled: true
     compression: zstd
 
+  prometheusremotewrite/observe:
+    endpoint: ${env:OBSERVE_PROMETHEUS_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+    resource_to_telemetry_conversion:
+      enabled: true  # Convert resource attributes to metric labels
+    send_metadata: true
+
   debug:
 
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
-      processors: [resourcedetection, resourcedetection/cloud]
-      exporters: [otlphttp/observe]
+      processors: [resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
     logs/forward:
       receivers: [otlp]

packaging/macos/connections/host_monitoring/host.yaml

@@ -9,4 +9,4 @@ service:
     metrics/agent-filestats:
        receivers: [filestats/agent]
        processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
+       exporters: [prometheusremotewrite/observe]

packaging/macos/connections/host_monitoring/host_metrics.yaml

@@ -39,4 +39,4 @@ service:
     metrics/host_monitoring_host:
       receivers: [hostmetrics/host-monitoring-host]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/macos/connections/host_monitoring/process_metrics.yaml

@@ -30,4 +30,4 @@ service:
     metrics/host_monitoring_process:
       receivers: [hostmetrics/host-monitoring-process]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/macos/connections/self_monitoring/logs_and_metrics.yaml

@@ -22,10 +22,10 @@ service:
   pipelines:
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
-      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
     logs/agent-config:
        receivers: [filelog/agent-config]
        processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
-       exporters: [otlphttp/observe]
+       exporters: [prometheusremotewrite/observe]

packaging/windows/config/otel-collector.yaml

@@ -22,7 +22,7 @@ processors:
   transform/truncate:
     log_statements:
       - context: log
-        statements: 
+        statements:
           - truncate_all(attributes, 4095)
           - truncate_all(resource.attributes, 4095)
     trace_statements:
@@ -35,17 +35,19 @@ processors:
     check_interval: 1s
     limit_percentage: 20
     spike_limit_percentage: 5
-  
+
   batch:
-
+
+  deltatocumulative:
+
   resourcedetection:
     detectors: [env, system]
     system:
       hostname_sources: ["os"]
       resource_attributes:
         host.id:
           enabled: true
-  
+
   resourcedetection/cloud:
     detectors:
     {{- if .CloudResourceDetectors }}
@@ -73,16 +75,24 @@ exporters:
       enabled: true
     compression: zstd
 
+  prometheusremotewrite/observe:
+    endpoint: ${env:OBSERVE_PROMETHEUS_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+    resource_to_telemetry_conversion:
+      enabled: true  # Convert resource attributes to metric labels
+    send_metadata: true
+
   debug:
 
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
-      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
-    logs/forward: 
+    logs/forward:
       receivers: [otlp]
       processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
       exporters: [otlphttp/observe, count]
@@ -91,7 +101,7 @@ service:
       receivers: [otlp]
       processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
       exporters: [otlphttp/observe]
-    
+
     metrics/count-nooop:
       receivers: [count]
       processors: [filter/count]

packaging/windows/connections/host_monitoring/host.yaml

@@ -9,4 +9,4 @@ service:
     metrics/agent-filestats:
        receivers: [filestats/agent]
        processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
+       exporters: [prometheusremotewrite/observe]

packaging/windows/connections/host_monitoring/host_metrics.yaml

@@ -27,4 +27,4 @@ service:
     metrics/host_monitoring_host:
       receivers: [hostmetrics/host-monitoring-host]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/windows/connections/host_monitoring/logs.yaml

@@ -8,7 +8,7 @@ receivers:
     channel: Security
     retry_on_failure:
       enabled: true
-      
+
   windowseventlog/host_monitoring-system:
     channel: System
     retry_on_failure:

packaging/windows/connections/host_monitoring/process_metrics.yaml

@@ -9,15 +9,15 @@ receivers:
           process.cpu.utilization:
             enabled: true
           process.disk.operations:
-            enabled: true      
+            enabled: true
           process.memory.utilization:
-            enabled: true      
+            enabled: true
           process.open_file_descriptors:
-            enabled: true      
+            enabled: true
           process.paging.faults:
-            enabled: true      
+            enabled: true
           process.signals_pending:
-            enabled: true      
+            enabled: true
           process.threads:
             enabled: true
         mute_process_name_error: true
@@ -30,4 +30,4 @@ service:
     metrics/host_monitoring_process:
       receivers: [hostmetrics/host-monitoring-process]
       processors: [memory_limiter, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      exporters: [prometheusremotewrite/observe]

packaging/windows/connections/self_monitoring/logs_and_metrics.yaml

@@ -22,8 +22,8 @@ service:
   pipelines:
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
-      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]
-      exporters: [otlphttp/observe]
+      processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch, deltatocumulative]
+      exporters: [prometheusremotewrite/observe]
 
     logs/agent-config:
        receivers: [filelog/agent-config]