feat: add option to send forwarded metrics to Observe via OTLP

obs-gh-mattcotter · obs-gh-mattcotter · commit b52a37e460db · 2025-05-19T10:25:12.000-05:00
diff --git a/internal/commands/initconfig/initconfig.go b/internal/commands/initconfig/initconfig.go
@@ -19,6 +19,7 @@ var (
 	observe_url                             string
 	cloud_resource_detectors                []string
 	resource_attributes                     map[string]string
+	forwarding_use_otlp                     bool
 	self_monitoring_enabled                 bool
 	host_monitoring_enabled                 bool
 	host_monitoring_logs_enabled            bool
@@ -87,6 +88,9 @@ func RegisterConfigFlags(cmd *cobra.Command, v *viper.Viper) {
 	cmd.PersistentFlags().StringToStringVar(&resource_attributes, "resource_attributes", map[string]string{}, "The cloud environments from which to detect resources")
 	v.BindPFlag("resource_attributes", cmd.PersistentFlags().Lookup("resource_attributes"))
 
+	cmd.PersistentFlags().BoolVar(&self_monitoring_enabled, "forwarding::otlp_metrics", false, "Forward app metrics to Observe using OTLP")
+	v.BindPFlag("forwarding::otlp_metrics", cmd.PersistentFlags().Lookup("forwarding::otlp_metrics"))
+
 	cmd.PersistentFlags().BoolVar(&self_monitoring_enabled, "self_monitoring::enabled", true, "Enable self monitoring")
 	v.BindPFlag("self_monitoring::enabled", cmd.PersistentFlags().Lookup("self_monitoring::enabled"))
 	v.SetDefault("self_monitoring::enabled", true)
diff --git a/internal/config/configschema.go b/internal/config/configschema.go
@@ -47,7 +47,8 @@ type HealthCheckConfig struct {
 }
 
 type ForwardingConfig struct {
-	Enabled bool `yaml:"enabled" mapstructure:"enabled" default:"true"`
+	Enabled     bool `yaml:"enabled" mapstructure:"enabled" default:"true"`
+	OtlpMetrics bool `yaml:"otlp_metrics,omitempty" mapstructure:"otlp_metrics"`
 }
 
 type InternalTelemetryMetricsConfig struct {
diff --git a/packaging/docker/observe-agent/connections/common/forward.yaml.tmpl b/packaging/docker/observe-agent/connections/common/forward.yaml.tmpl
@@ -19,11 +19,24 @@ exporters:
       enabled: true
     compression: zstd
 
+  otlphttp/observemetrics:
+    endpoint: ${env:OBSERVE_OTEL_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+      x-observe-target-package: "Metrics"
+    sending_queue:
+      num_consumers: 4
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+    compression: zstd
+
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -32,13 +45,22 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        {{- if not .Forwarding.OtlpMetrics }}
         - deltatocumulative
+        {{- end }}
         - batch
-      exporters: [prometheusremotewrite/observe]
+      exporters:
+        {{- if .Forwarding.OtlpMetrics }}
+        - otlphttp/observemetrics
+        {{- else }}
+        - prometheusremotewrite/observe
+        {{- end }}
 
     logs/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
+        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -47,11 +69,14 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        - batch
       exporters: [otlphttp/observe, count]
 
     traces/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
+        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -60,4 +85,5 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        - batch
       exporters: [otlphttp/observetracing]
diff --git a/packaging/linux/config/observe-agent.yaml b/packaging/linux/config/observe-agent.yaml
@@ -7,6 +7,11 @@ observe_url: "<OBSERVE COLLECTION ENDPOINT>"
 # Debug mode - Sets agent log level to debug
 debug: false
 
+# Enable forwarding of local app metrics and traces
+forwarding:
+  enabled: true
+  otlp_metrics: true
+
 # collect metrics and logs pertaining to the agent itself
 self_monitoring:
   enabled: true
diff --git a/packaging/linux/connections/common/forward.yaml.tmpl b/packaging/linux/connections/common/forward.yaml.tmpl
@@ -19,11 +19,24 @@ exporters:
       enabled: true
     compression: zstd
 
+  otlphttp/observemetrics:
+    endpoint: ${env:OBSERVE_OTEL_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+      x-observe-target-package: "Metrics"
+    sending_queue:
+      num_consumers: 4
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+    compression: zstd
+
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -32,13 +45,22 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        {{- if not .Forwarding.OtlpMetrics }}
         - deltatocumulative
+        {{- end }}
         - batch
-      exporters: [prometheusremotewrite/observe]
+      exporters:
+        {{- if .Forwarding.OtlpMetrics }}
+        - otlphttp/observemetrics
+        {{- else }}
+        - prometheusremotewrite/observe
+        {{- end }}
 
     logs/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
+        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -47,11 +69,14 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        - batch
       exporters: [otlphttp/observe, count]
 
     traces/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
+        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -60,4 +85,5 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        - batch
       exporters: [otlphttp/observetracing]
diff --git a/packaging/macos/connections/common/forward.yaml.tmpl b/packaging/macos/connections/common/forward.yaml.tmpl
@@ -19,11 +19,24 @@ exporters:
       enabled: true
     compression: zstd
 
+  otlphttp/observemetrics:
+    endpoint: ${env:OBSERVE_OTEL_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+      x-observe-target-package: "Metrics"
+    sending_queue:
+      num_consumers: 4
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+    compression: zstd
+
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -32,13 +45,22 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        {{- if not .Forwarding.OtlpMetrics }}
         - deltatocumulative
+        {{- end }}
         - batch
-      exporters: [prometheusremotewrite/observe]
+      exporters:
+        {{- if .Forwarding.OtlpMetrics }}
+        - otlphttp/observemetrics
+        {{- else }}
+        - prometheusremotewrite/observe
+        {{- end }}
 
     logs/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
+        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -47,11 +69,14 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        - batch
       exporters: [otlphttp/observe, count]
 
     traces/forward:
       receivers: [otlp]
       processors:
+        - memory_limiter
+        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -60,4 +85,5 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        - batch
       exporters: [otlphttp/observetracing]
diff --git a/packaging/windows/connections/common/forward.yaml.tmpl b/packaging/windows/connections/common/forward.yaml.tmpl
@@ -19,13 +19,24 @@ exporters:
       enabled: true
     compression: zstd
 
+  otlphttp/observemetrics:
+    endpoint: ${env:OBSERVE_OTEL_ENDPOINT}
+    headers:
+      authorization: ${env:OBSERVE_AUTHORIZATION_HEADER}
+      x-observe-target-package: "Metrics"
+    sending_queue:
+      num_consumers: 4
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+    compression: zstd
+
 service:
   pipelines:
     metrics/forward:
       receivers: [otlp]
       processors:
         - memory_limiter
-        - transform/truncate
         - resourcedetection
         - resourcedetection/cloud
         {{- if .HasAttributes }}
@@ -34,9 +45,16 @@ service:
         {{- if .HasResourceAttributes }}
         - resource/observe_global_resource_attributes
         {{- end }}
+        {{- if not .Forwarding.OtlpMetrics }}
         - deltatocumulative
+        {{- end }}
         - batch
-      exporters: [prometheusremotewrite/observe]
+      exporters:
+        {{- if .Forwarding.OtlpMetrics }}
+        - otlphttp/observemetrics
+        {{- else }}
+        - prometheusremotewrite/observe
+        {{- end }}
 
     logs/forward:
       receivers: [otlp]
diff --git a/packaging/windows/observe-agent.yaml b/packaging/windows/observe-agent.yaml
@@ -7,6 +7,11 @@ observe_url: "${OBSERVE_COLLECTION_ENDPOINT}"
 # Debug mode - Sets agent log level to debug
 debug: false
 
+# Enable forwarding of local app metrics and traces
+forwarding:
+  enabled: true
+  otlp_metrics: true
+
 # collect metrics and logs pertaining to the agent itself
 self_monitoring:
   enabled: true
diff --git a/scripts/install_linux.sh b/scripts/install_linux.sh
@@ -120,6 +120,8 @@ else
             INIT_FLAGS="$INIT_FLAGS --host_monitoring::metrics::host::enabled=false"
         fi
     fi
+    # Add default args
+    INIT_FLAGS="$INIT_FLAGS --forwarding::otlp_metrics=true"
     sudo $agent_binary_path init-config $INIT_FLAGS
     sudo chown root:root $observeagent_config_dir/observe-agent.yaml
 fi
diff --git a/scripts/install_mac.sh b/scripts/install_mac.sh
@@ -127,6 +127,8 @@ else
             INIT_FLAGS="$INIT_FLAGS --host_monitoring::metrics::host::enabled=false"
         fi
     fi
+    # Add default args
+    INIT_FLAGS="$INIT_FLAGS --forwarding::otlp_metrics=true"
     sudo $observeagent_install_dir/observe-agent init-config $INIT_FLAGS
     sudo chown root:wheel $observeagent_install_dir/observe-agent.yaml
 fi

Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,8 @@ type HealthCheckConfig struct {`
`47`	`47`	`}`
`48`	`48`
`49`	`49`	`type ForwardingConfig struct {`
`50`		- Enabled bool `yaml:"enabled" mapstructure:"enabled" default:"true"`
	`50`	+ Enabled bool `yaml:"enabled" mapstructure:"enabled" default:"true"`
	`51`	+ OtlpMetrics bool `yaml:"otlp_metrics,omitempty" mapstructure:"otlp_metrics"`
`51`	`52`	`}`
`52`	`53`
`53`	`54`	`type InternalTelemetryMetricsConfig struct {`