From 78f61a0f54a47d764ccb326e7fb8ae7797dcac87 Mon Sep 17 00:00:00 2001 From: Matt Cotter Date: Wed, 25 Jun 2025 12:19:20 -0500 Subject: [PATCH] refactor: remove all receiver references to bundled config host files --- go.mod | 4 +-- internal/commands/config/config_test.go | 3 +++ .../config/test/snap1-docker-output.yaml | 22 +++------------- .../config/test/snap1-linux-output.yaml | 22 +++------------- .../config/test/snap1-macos-output.yaml | 22 +++------------- .../config/test/snap1-windows-output.yaml | 22 +++------------- internal/connections/allconnectiontypes.go | 1 + internal/root/root.go | 5 ++++ .../host_monitoring/host.yaml.tmpl | 3 +-- .../logs_and_metrics.yaml.tmpl | 26 +++---------------- .../host_monitoring/host.yaml.tmpl | 3 +-- .../logs_and_metrics.yaml.tmpl | 26 +++---------------- .../host_monitoring/host.yaml.tmpl | 3 +-- .../logs_and_metrics.yaml.tmpl | 26 +++---------------- .../host_monitoring/host.yaml.tmpl | 3 +-- .../logs_and_metrics.yaml.tmpl | 26 +++---------------- 16 files changed, 43 insertions(+), 174 deletions(-) diff --git a/go.mod b/go.mod index 5dcde07f2..2ea17e1c5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23.10 require ( github.com/go-viper/mapstructure/v2 v2.2.1 + github.com/invopop/jsonschema v0.13.0 github.com/jarcoal/httpmock v1.3.1 github.com/mcuadros/go-defaults v1.2.0 github.com/observeinc/observe-agent/observecol v0.0.0-00010101000000-000000000000 @@ -13,6 +14,7 @@ require ( github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/spf13/viper v1.20.0-alpha.6 + github.com/stoewer/go-strcase v1.3.1 github.com/stretchr/testify v1.10.0 go.opentelemetry.io/collector/confmap v1.34.0 go.opentelemetry.io/collector/otelcol v0.128.0 @@ -174,7 +176,6 @@ require ( github.com/hetznercloud/hcloud-go/v2 v2.21.0 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/jsonschema v0.13.0 // indirect github.com/ionos-cloud/sdk-go/v6 v6.3.3 // indirect github.com/jaegertracing/jaeger-idl v0.5.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -337,7 +338,6 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/thda/tds v0.1.7 // indirect diff --git a/internal/commands/config/config_test.go b/internal/commands/config/config_test.go index c544b5940..0b32b1f20 100644 --- a/internal/commands/config/config_test.go +++ b/internal/commands/config/config_test.go @@ -128,10 +128,13 @@ func setEnvVars(t *testing.T, packageType PackageType) { switch packageType { case MacOS: assert.NoError(t, os.Setenv("FILESTORAGE_PATH", "/var/lib/observe-agent/filestorage")) + assert.NoError(t, os.Setenv("OBSERVE_AGENT_CONFIG_PATH", "/usr/local/observe-agent/observe-agent.yaml")) case Windows: assert.NoError(t, os.Setenv("FILESTORAGE_PATH", "C:\\ProgramData\\Observe\\observe-agent\\filestorage")) + assert.NoError(t, os.Setenv("OBSERVE_AGENT_CONFIG_PATH", "C:\\Program Files\\Observe\\observe-agent\\observe-agent.yaml")) case Linux, Docker: assert.NoError(t, os.Setenv("FILESTORAGE_PATH", "/var/lib/observe-agent/filestorage")) + assert.NoError(t, os.Setenv("OBSERVE_AGENT_CONFIG_PATH", "/etc/observe-agent/observe-agent.yaml")) default: t.Errorf("Unknown package type: %s", packageType) } diff --git a/internal/commands/config/test/snap1-docker-output.yaml b/internal/commands/config/test/snap1-docker-output.yaml index e9e651a12..d5a11ab89 100644 --- a/internal/commands/config/test/snap1-docker-output.yaml +++ b/internal/commands/config/test/snap1-docker-output.yaml @@ -131,14 +131,6 @@ processors: - truncate_all(attributes, 4095) - truncate_all(resource.attributes, 4095) receivers: - filelog/agent-config: - include: - - /etc/observe-agent/connections/common/base.yaml.tmpl - include_file_path: true - multiline: - line_end_pattern: ENDOFLINEPATTERN - poll_interval: 5m - start_at: beginning filelog/host_monitoring: exclude: - exclude1 @@ -160,7 +152,7 @@ receivers: storage: file_storage filestats/agent: collection_interval: 5m - include: /etc/observe-agent/connections/common/base.yaml.tmpl + include: /etc/observe-agent/observe-agent.yaml initial_delay: 60s hostmetrics/host-monitoring-host: collection_interval: 60s @@ -264,17 +256,9 @@ service: pipelines: logs/agent-config: exporters: - - otlphttp/observe - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - - attributes/observe_global_attributes - - resource/observe_global_resource_attributes - - batch + - nop receivers: - - filelog/agent-config + - nop logs/agent-journald: exporters: - otlphttp/observe diff --git a/internal/commands/config/test/snap1-linux-output.yaml b/internal/commands/config/test/snap1-linux-output.yaml index ecb954dcf..2affabc69 100644 --- a/internal/commands/config/test/snap1-linux-output.yaml +++ b/internal/commands/config/test/snap1-linux-output.yaml @@ -128,14 +128,6 @@ processors: - truncate_all(attributes, 4095) - truncate_all(resource.attributes, 4095) receivers: - filelog/agent-config: - include: - - /etc/observe-agent/connections/common/base.yaml.tmpl - include_file_path: true - multiline: - line_end_pattern: ENDOFLINEPATTERN - poll_interval: 5m - start_at: beginning filelog/host_monitoring: exclude: - exclude1 @@ -157,7 +149,7 @@ receivers: storage: file_storage filestats/agent: collection_interval: 5m - include: /etc/observe-agent/connections/common/base.yaml.tmpl + include: /etc/observe-agent/observe-agent.yaml initial_delay: 60s hostmetrics/host-monitoring-host: collection_interval: 60s @@ -258,17 +250,9 @@ service: pipelines: logs/agent-config: exporters: - - otlphttp/observe - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - - attributes/observe_global_attributes - - resource/observe_global_resource_attributes - - batch + - nop receivers: - - filelog/agent-config + - nop logs/agent-journald: exporters: - otlphttp/observe diff --git a/internal/commands/config/test/snap1-macos-output.yaml b/internal/commands/config/test/snap1-macos-output.yaml index c1243bcb5..0e1a7b623 100644 --- a/internal/commands/config/test/snap1-macos-output.yaml +++ b/internal/commands/config/test/snap1-macos-output.yaml @@ -128,14 +128,6 @@ processors: - truncate_all(attributes, 4095) - truncate_all(resource.attributes, 4095) receivers: - filelog/agent-config: - include: - - /usr/local/observe-agent/connections/common/base.yaml.tmpl - include_file_path: true - multiline: - line_end_pattern: ENDOFLINEPATTERN - poll_interval: 5m - start_at: beginning filelog/host_monitoring: exclude: - exclude1 @@ -157,7 +149,7 @@ receivers: storage: file_storage filestats/agent: collection_interval: 5m - include: /usr/local/observe-agent/connections/common/base.yaml.tmpl + include: /usr/local/observe-agent/observe-agent.yaml initial_delay: 60s hostmetrics/host-monitoring-host: collection_interval: 60s @@ -245,17 +237,9 @@ service: pipelines: logs/agent-config: exporters: - - otlphttp/observe - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - - attributes/observe_global_attributes - - resource/observe_global_resource_attributes - - batch + - nop receivers: - - filelog/agent-config + - nop logs/forward: exporters: - otlphttp/observe diff --git a/internal/commands/config/test/snap1-windows-output.yaml b/internal/commands/config/test/snap1-windows-output.yaml index 24af4094a..802138eac 100644 --- a/internal/commands/config/test/snap1-windows-output.yaml +++ b/internal/commands/config/test/snap1-windows-output.yaml @@ -107,14 +107,6 @@ processors: - truncate_all(attributes, 4095) - truncate_all(resource.attributes, 4095) receivers: - filelog/agent-config: - include: - - C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl - include_file_path: true - multiline: - line_end_pattern: ENDOFLINEPATTERN - poll_interval: 5m - start_at: beginning filelog/host_monitoring: exclude: - exclude1 @@ -136,7 +128,7 @@ receivers: storage: file_storage filestats/agent: collection_interval: 5m - include: C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl + include: C:\Program Files\Observe\observe-agent\observe-agent.yaml initial_delay: 60s hostmetrics/host-monitoring-host: collection_interval: 60s @@ -224,17 +216,9 @@ service: pipelines: logs/agent-config: exporters: - - otlphttp/observe - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - - attributes/observe_global_attributes - - resource/observe_global_resource_attributes - - batch + - nop receivers: - - filelog/agent-config + - nop logs/forward: exporters: - otlphttp/observe diff --git a/internal/connections/allconnectiontypes.go b/internal/connections/allconnectiontypes.go index ddd4cc7e1..4df798474 100644 --- a/internal/connections/allconnectiontypes.go +++ b/internal/connections/allconnectiontypes.go @@ -58,6 +58,7 @@ var HostMonitoringConnectionType = MakeConnectionType( []CollectorConfigFragment{ { enabledCheck: func(agentConfig *config.AgentConfig) bool { + // TODO remove this deprecated template return agentConfig.HostMonitoring.Enabled }, colConfigFilePath: "host.yaml.tmpl", diff --git a/internal/root/root.go b/internal/root/root.go index 7c40fe57f..169338395 100644 --- a/internal/root/root.go +++ b/internal/root/root.go @@ -104,6 +104,11 @@ func setEnvVars() error { os.Setenv("OBSERVE_AUTHORIZATION_HEADER", "Bearer "+token) os.Setenv("FILESTORAGE_PATH", getDefaultFilestoragePath()) + configFile := viper.ConfigFileUsed() + if configFile != "" { + os.Setenv("OBSERVE_AGENT_CONFIG_PATH", configFile) + } + // Default TRACE_TOKEN to be the value of the configured token if it's not set. This allows for users to upgrade to // direct write tracing with ingest tokens in kubernetes without breaking backwards compatibility in our helm chart. // TODO: remove this once our helm chart no longer supports TRACE_TOKEN diff --git a/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl b/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl index 8c0e3257d..b45bce46c 100644 --- a/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl +++ b/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl @@ -1,7 +1,6 @@ receivers: - # TODO deprecate this filestats/agent: - include: '/etc/observe-agent/connections/common/base.yaml.tmpl' + include: "${env:OBSERVE_AGENT_CONFIG_PATH}" collection_interval: 5m initial_delay: 60s diff --git a/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 88cecd34a..031b6b972 100644 --- a/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,12 +1,5 @@ receivers: - filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - # TODO deprecate this - include: [/etc/observe-agent/connections/common/base.yaml.tmpl] - include_file_path: true - start_at: beginning - poll_interval: 5m - multiline: - line_end_pattern: ENDOFLINEPATTERN + # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields prometheus/agent: config: @@ -60,18 +53,7 @@ service: - batch exporters: [otlphttp/observe, count] + # Deprecated logs/agent-config: - receivers: [filelog/agent-config] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe] + receivers: [nop] + exporters: [nop] diff --git a/packaging/linux/connections/host_monitoring/host.yaml.tmpl b/packaging/linux/connections/host_monitoring/host.yaml.tmpl index 8c0e3257d..b45bce46c 100644 --- a/packaging/linux/connections/host_monitoring/host.yaml.tmpl +++ b/packaging/linux/connections/host_monitoring/host.yaml.tmpl @@ -1,7 +1,6 @@ receivers: - # TODO deprecate this filestats/agent: - include: '/etc/observe-agent/connections/common/base.yaml.tmpl' + include: "${env:OBSERVE_AGENT_CONFIG_PATH}" collection_interval: 5m initial_delay: 60s diff --git a/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 88cecd34a..031b6b972 100644 --- a/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,12 +1,5 @@ receivers: - filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - # TODO deprecate this - include: [/etc/observe-agent/connections/common/base.yaml.tmpl] - include_file_path: true - start_at: beginning - poll_interval: 5m - multiline: - line_end_pattern: ENDOFLINEPATTERN + # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields prometheus/agent: config: @@ -60,18 +53,7 @@ service: - batch exporters: [otlphttp/observe, count] + # Deprecated logs/agent-config: - receivers: [filelog/agent-config] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe] + receivers: [nop] + exporters: [nop] diff --git a/packaging/macos/connections/host_monitoring/host.yaml.tmpl b/packaging/macos/connections/host_monitoring/host.yaml.tmpl index 79fc44a7a..b45bce46c 100644 --- a/packaging/macos/connections/host_monitoring/host.yaml.tmpl +++ b/packaging/macos/connections/host_monitoring/host.yaml.tmpl @@ -1,7 +1,6 @@ receivers: - # TODO deprecate this filestats/agent: - include: '/usr/local/observe-agent/connections/common/base.yaml.tmpl' + include: "${env:OBSERVE_AGENT_CONFIG_PATH}" collection_interval: 5m initial_delay: 60s diff --git a/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 4c3aab8d3..e45c0aae1 100644 --- a/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,12 +1,5 @@ receivers: - filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - # TODO deprecate this - include: [/usr/local/observe-agent/connections/common/base.yaml.tmpl] - include_file_path: true - start_at: beginning - poll_interval: 5m - multiline: - line_end_pattern: ENDOFLINEPATTERN + # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields prometheus/agent: config: @@ -39,18 +32,7 @@ service: - batch exporters: [prometheusremotewrite/observe] + # Deprecated logs/agent-config: - receivers: [filelog/agent-config] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe] + receivers: [nop] + exporters: [nop] diff --git a/packaging/windows/connections/host_monitoring/host.yaml.tmpl b/packaging/windows/connections/host_monitoring/host.yaml.tmpl index ade69d1e7..b45bce46c 100644 --- a/packaging/windows/connections/host_monitoring/host.yaml.tmpl +++ b/packaging/windows/connections/host_monitoring/host.yaml.tmpl @@ -1,7 +1,6 @@ receivers: - # TODO deprecate this filestats/agent: - include: 'C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl' + include: "${env:OBSERVE_AGENT_CONFIG_PATH}" collection_interval: 5m initial_delay: 60s diff --git a/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 209afa1b6..e45c0aae1 100644 --- a/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,12 +1,5 @@ receivers: - filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - # TODO deprecate this - include: ['C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl'] - include_file_path: true - start_at: beginning - poll_interval: 5m - multiline: - line_end_pattern: ENDOFLINEPATTERN + # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields prometheus/agent: config: @@ -39,18 +32,7 @@ service: - batch exporters: [prometheusremotewrite/observe] + # Deprecated logs/agent-config: - receivers: [filelog/agent-config] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe] + receivers: [nop] + exporters: [nop]