From 35411eced6f7e3b4f159ed0c6803361cda433cdd Mon Sep 17 00:00:00 2001 From: Matt Cotter Date: Tue, 24 Jun 2025 15:40:43 -0500 Subject: [PATCH] feat: add all bundled otel config to the observe-agent binary instead of relying on files on disk --- .github/workflows/release-build.yaml | 2 +- .github/workflows/release-nightly.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/tests-integration.yaml | 2 +- .goreleaser.yaml | 22 ++-- Makefile | 2 +- integration/scripts/install_windows.ps1 | 1 - internal/commands/config/config_test.go | 19 +-- internal/commands/status/status.go | 2 +- internal/connections/allconnectiontypes.go | 6 +- .../bundledconfig/bundledconfig.go | 46 +++++++ .../docker}/common/base.yaml.tmpl | 1 + .../host_monitoring/host_metrics.yaml.tmpl | 2 + .../docker}/host_monitoring/logs.yaml.tmpl | 1 + .../host_monitoring/process_metrics.yaml.tmpl | 1 + .../logs_and_metrics.yaml.tmpl | 12 +- .../bundledconfig/docker/templates.go | 16 +++ .../bundledconfig/docker_defaults.go | 9 ++ .../host_monitoring/host_metrics.yaml.tmpl | 2 + .../linux}/host_monitoring/logs.yaml.tmpl | 1 + .../logs_and_metrics.yaml.tmpl | 12 +- .../bundledconfig/linux/templates.go | 12 ++ .../bundledconfig/linux_defaults.go | 9 ++ .../connections/bundledconfig/mac_defaults.go | 9 ++ .../bundledconfig/macos/templates.go | 3 + .../shared}/common/attributes.yaml.tmpl | 0 .../shared}/common/base.yaml.tmpl | 1 + .../shared}/common/forward.yaml.tmpl | 0 .../shared}/common/health_check.yaml.tmpl | 0 .../common/internal_telemetry.yaml.tmpl | 0 .../shared}/host_monitoring/host.yaml.tmpl | 0 .../host_monitoring/host_metrics.yaml.tmpl | 0 .../shared}/host_monitoring/logs.yaml.tmpl | 0 .../host_monitoring/process_metrics.yaml.tmpl | 0 .../logs_and_metrics.yaml.tmpl | 1 + .../bundledconfig/shared/templates.go | 26 ++++ .../windows}/common/base.yaml.tmpl | 1 + .../host_monitoring/host_metrics.yaml.tmpl | 1 + .../windows}/host_monitoring/logs.yaml.tmpl | 1 + .../bundledconfig/windows/templates.go | 12 ++ .../bundledconfig/windows_defaults.go | 9 ++ internal/connections/confighandler.go | 12 +- internal/connections/connections.go | 82 +++++++------ internal/connections/connections_test.go | 36 ++++-- internal/connections/templatefuncs.go | 26 ++-- .../connections/common/attributes.yaml.tmpl | 19 --- .../linux/connections/common/base.yaml.tmpl | 116 ------------------ .../connections/common/forward.yaml.tmpl | 89 -------------- .../connections/common/health_check.yaml.tmpl | 4 - .../common/internal_telemetry.yaml.tmpl | 16 --- .../host_monitoring/host.yaml.tmpl | 22 ---- .../connections/common/attributes.yaml.tmpl | 19 --- .../connections/common/forward.yaml.tmpl | 89 -------------- .../connections/common/health_check.yaml.tmpl | 4 - .../common/internal_telemetry.yaml.tmpl | 16 --- .../host_monitoring/host.yaml.tmpl | 22 ---- .../host_monitoring/process_metrics.yaml.tmpl | 43 ------- .../connections/common/attributes.yaml.tmpl | 19 --- .../connections/common/forward.yaml.tmpl | 89 -------------- .../connections/common/health_check.yaml.tmpl | 4 - .../common/internal_telemetry.yaml.tmpl | 16 --- .../host_monitoring/host.yaml.tmpl | 22 ---- .../host_monitoring/process_metrics.yaml.tmpl | 43 ------- .../logs_and_metrics.yaml.tmpl | 38 ------ scripts/install.ps1 | 6 +- scripts/install_linux.sh | 4 +- scripts/install_mac.sh | 4 +- 67 files changed, 311 insertions(+), 797 deletions(-) create mode 100644 internal/connections/bundledconfig/bundledconfig.go rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/docker}/common/base.yaml.tmpl (97%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/docker}/host_monitoring/host_metrics.yaml.tmpl (89%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/docker}/host_monitoring/logs.yaml.tmpl (96%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/docker}/host_monitoring/process_metrics.yaml.tmpl (95%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/docker}/self_monitoring/logs_and_metrics.yaml.tmpl (91%) create mode 100644 internal/connections/bundledconfig/docker/templates.go create mode 100644 internal/connections/bundledconfig/docker_defaults.go rename {packaging/linux/connections => internal/connections/bundledconfig/linux}/host_monitoring/host_metrics.yaml.tmpl (88%) rename {packaging/linux/connections => internal/connections/bundledconfig/linux}/host_monitoring/logs.yaml.tmpl (97%) rename {packaging/linux/connections => internal/connections/bundledconfig/linux}/self_monitoring/logs_and_metrics.yaml.tmpl (91%) create mode 100644 internal/connections/bundledconfig/linux/templates.go create mode 100644 internal/connections/bundledconfig/linux_defaults.go create mode 100644 internal/connections/bundledconfig/mac_defaults.go create mode 100644 internal/connections/bundledconfig/macos/templates.go rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/shared}/common/attributes.yaml.tmpl (100%) rename {packaging/macos/connections => internal/connections/bundledconfig/shared}/common/base.yaml.tmpl (97%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/shared}/common/forward.yaml.tmpl (100%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/shared}/common/health_check.yaml.tmpl (100%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/shared}/common/internal_telemetry.yaml.tmpl (100%) rename {packaging/docker/observe-agent/connections => internal/connections/bundledconfig/shared}/host_monitoring/host.yaml.tmpl (100%) rename {packaging/macos/connections => internal/connections/bundledconfig/shared}/host_monitoring/host_metrics.yaml.tmpl (100%) rename {packaging/macos/connections => internal/connections/bundledconfig/shared}/host_monitoring/logs.yaml.tmpl (100%) rename {packaging/linux/connections => internal/connections/bundledconfig/shared}/host_monitoring/process_metrics.yaml.tmpl (100%) rename {packaging/macos/connections => internal/connections/bundledconfig/shared}/self_monitoring/logs_and_metrics.yaml.tmpl (93%) create mode 100644 internal/connections/bundledconfig/shared/templates.go rename {packaging/windows/connections => internal/connections/bundledconfig/windows}/common/base.yaml.tmpl (96%) rename {packaging/windows/connections => internal/connections/bundledconfig/windows}/host_monitoring/host_metrics.yaml.tmpl (91%) rename {packaging/windows/connections => internal/connections/bundledconfig/windows}/host_monitoring/logs.yaml.tmpl (97%) create mode 100644 internal/connections/bundledconfig/windows/templates.go create mode 100644 internal/connections/bundledconfig/windows_defaults.go delete mode 100644 packaging/linux/connections/common/attributes.yaml.tmpl delete mode 100644 packaging/linux/connections/common/base.yaml.tmpl delete mode 100644 packaging/linux/connections/common/forward.yaml.tmpl delete mode 100644 packaging/linux/connections/common/health_check.yaml.tmpl delete mode 100644 packaging/linux/connections/common/internal_telemetry.yaml.tmpl delete mode 100644 packaging/linux/connections/host_monitoring/host.yaml.tmpl delete mode 100644 packaging/macos/connections/common/attributes.yaml.tmpl delete mode 100644 packaging/macos/connections/common/forward.yaml.tmpl delete mode 100644 packaging/macos/connections/common/health_check.yaml.tmpl delete mode 100644 packaging/macos/connections/common/internal_telemetry.yaml.tmpl delete mode 100644 packaging/macos/connections/host_monitoring/host.yaml.tmpl delete mode 100644 packaging/macos/connections/host_monitoring/process_metrics.yaml.tmpl delete mode 100644 packaging/windows/connections/common/attributes.yaml.tmpl delete mode 100644 packaging/windows/connections/common/forward.yaml.tmpl delete mode 100644 packaging/windows/connections/common/health_check.yaml.tmpl delete mode 100644 packaging/windows/connections/common/internal_telemetry.yaml.tmpl delete mode 100644 packaging/windows/connections/host_monitoring/host.yaml.tmpl delete mode 100644 packaging/windows/connections/host_monitoring/process_metrics.yaml.tmpl delete mode 100644 packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl diff --git a/.github/workflows/release-build.yaml b/.github/workflows/release-build.yaml index 14b3b8ca6..07bb187b2 100644 --- a/.github/workflows/release-build.yaml +++ b/.github/workflows/release-build.yaml @@ -29,7 +29,7 @@ jobs: - name: Install qemu uses: docker/setup-qemu-action@v3 - name: Run GoReleaser - timeout-minutes: 18 + timeout-minutes: 25 uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro diff --git a/.github/workflows/release-nightly.yaml b/.github/workflows/release-nightly.yaml index 0fd946508..acd165f56 100644 --- a/.github/workflows/release-nightly.yaml +++ b/.github/workflows/release-nightly.yaml @@ -61,7 +61,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 - name: Run GoReleaser - timeout-minutes: 18 + timeout-minutes: 25 uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3b764adf9..b7be60852 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -56,7 +56,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 - name: Run GoReleaser - timeout-minutes: 18 + timeout-minutes: 25 uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 8f807ce30..102a5975e 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -37,7 +37,7 @@ jobs: - name: Install qemu uses: docker/setup-qemu-action@v3 - name: Run GoReleaser - timeout-minutes: 18 + timeout-minutes: 25 uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro diff --git a/.goreleaser.yaml b/.goreleaser.yaml index fe2b03cb8..e214c2b0f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -24,6 +24,16 @@ builds: - linux ldflags: -s -w -X github.com/observeinc/observe-agent/build.Version={{.Version}} + - id: docker_build + binary: observe-agent + env: + - CGO_ENABLED=0 + goos: + - linux + flags: + - -tags=docker + ldflags: -s -w -X github.com/observeinc/observe-agent/build.Version={{.Version}} + - id: mac_build binary: observe-agent env: @@ -59,8 +69,6 @@ archives: # The default agent config file and the service file live in config. - src: "packaging/linux/config/*" dst: "." - - src: "packaging/linux/connections/*" - dst: "connections" - id: windows formats: ["zip"] @@ -75,8 +83,6 @@ archives: files: - src: "packaging/windows/observe-agent.yaml" dst: "observe-agent.yaml" - - src: "packaging/windows/connections/*" - dst: "connections" - id: macos formats: ["zip"] @@ -120,6 +126,8 @@ changelog: nfpms: - id: linux + ids: [linux_build] + package_name: observe-agent # You can change the file name of the package. @@ -162,10 +170,6 @@ nfpms: # Contents to add to the package. # GoReleaser will automatically add the binaries. contents: - - src: packaging/linux/connections - dst: /etc/observe-agent/connections - type: tree - - src: packaging/linux/config/observe-agent.yaml dst: /etc/observe-agent/observe-agent.yaml type: config|noreplace @@ -200,6 +204,7 @@ dockers: use: buildx goos: linux goarch: amd64 + ids: [docker_build] build_flag_templates: - "--platform=linux/amd64" extra_files: @@ -211,6 +216,7 @@ dockers: use: buildx goos: linux goarch: arm64 + ids: [docker_build] build_flag_templates: - "--platform=linux/arm64/v8" extra_files: diff --git a/Makefile b/Makefile index c5008444f..fcd860fe2 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ build: go build ./... docker-image: - env GOOS=linux GOARCH=arm64 go build -o observe-agent + env GOOS=linux GOARCH=arm64 go build -tags docker -o observe-agent docker build -f packaging/docker/Dockerfile -t observe-agent:dev . ## test: Runs Go tests across all packages diff --git a/integration/scripts/install_windows.ps1 b/integration/scripts/install_windows.ps1 index ec5625b55..7ce5549ea 100644 --- a/integration/scripts/install_windows.ps1 +++ b/integration/scripts/install_windows.ps1 @@ -36,5 +36,4 @@ Expand-Archive -Force -LiteralPath $local_installer -DestinationPath "$temp_dir Write-Output "Copying files from $temp_dir\observe-agent_extract to $observeagent_install_dir" Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.exe -Destination $observeagent_install_dir Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.yaml -Destination $observeagent_install_dir -Copy-Item -Force -Path $temp_dir\observe-agent_extract\connections\* -Destination $observeagent_install_dir\connections -Recurse Get-ChildItem -Path $observeagent_install_dir -Recurse diff --git a/internal/commands/config/config_test.go b/internal/commands/config/config_test.go index f2a998544..f28e63035 100644 --- a/internal/commands/config/config_test.go +++ b/internal/commands/config/config_test.go @@ -6,6 +6,7 @@ package config import ( "bytes" "context" + "embed" "os" "path" "path/filepath" @@ -15,6 +16,7 @@ import ( "github.com/observeinc/observe-agent/internal/commands/util/logger" "github.com/observeinc/observe-agent/internal/connections" + "github.com/observeinc/observe-agent/internal/connections/bundledconfig" "github.com/observeinc/observe-agent/internal/root" "github.com/observeinc/observe-agent/observecol" "github.com/spf13/pflag" @@ -109,7 +111,7 @@ func runSnapshotTest(t *testing.T, test snapshotTest) { // Set the template base dir for all connections for _, conn := range connections.AllConnectionTypes { - conn.ApplyOptions(connections.WithConfigFolderPath(getPackagingPath(t, test.packageType, curPath))) + conn.ApplyOptions(connections.WithConfigTemplateOverrides(getTemplateOverrides(t, test.packageType, curPath))) } // Set config flags @@ -132,16 +134,19 @@ func runSnapshotTest(t *testing.T, test snapshotTest) { assert.Equal(t, strings.TrimSpace(string(expected)), strings.TrimSpace(output.String())) } -func getPackagingPath(t *testing.T, packageType PackageType, curPath string) string { - const packagingPath = "../../../packaging" +func getTemplateOverrides(t *testing.T, packageType PackageType, curPath string) map[string]embed.FS { switch packageType { - case MacOS, Linux, Windows: - return filepath.Join(curPath, packagingPath, string(packageType), "connections") + case MacOS: + return bundledconfig.MacOSTemplateFS + case Linux: + return bundledconfig.LinuxTemplateFS + case Windows: + return bundledconfig.WindowsTemplateFS case Docker: - return filepath.Join(curPath, packagingPath, "docker/observe-agent/connections") + return bundledconfig.DockerTemplateFS default: t.Errorf("Unknown package type: %s", packageType) - return "" + return nil } } diff --git a/internal/commands/status/status.go b/internal/commands/status/status.go index c34a74bf7..b70286a72 100644 --- a/internal/commands/status/status.go +++ b/internal/commands/status/status.go @@ -49,7 +49,7 @@ func getStatusFromTemplate(v *viper.Viper) error { return err } t := template.Must(template.New(statusTemplate). - Funcs(connections.GetTemplateFuncMap()). + Funcs(connections.TemplateFuncMap). ParseFS(statusTemplateFS, statusTemplate)) if err := t.ExecuteTemplate(os.Stdout, statusTemplate, data); err != nil { return err diff --git a/internal/connections/allconnectiontypes.go b/internal/connections/allconnectiontypes.go index 4df798474..9f462e71b 100644 --- a/internal/connections/allconnectiontypes.go +++ b/internal/connections/allconnectiontypes.go @@ -15,7 +15,7 @@ var CommonConnectionType = MakeConnectionType( func(_ *config.AgentConfig) bool { return true }, - []CollectorConfigFragment{ + []BundledConfigFragment{ { enabledCheck: func(_ *config.AgentConfig) bool { // Always include the base connection. @@ -55,7 +55,7 @@ var HostMonitoringConnectionType = MakeConnectionType( func(agentConfig *config.AgentConfig) bool { return agentConfig.HostMonitoring.Enabled }, - []CollectorConfigFragment{ + []BundledConfigFragment{ { enabledCheck: func(agentConfig *config.AgentConfig) bool { // TODO remove this deprecated template @@ -89,7 +89,7 @@ var SelfMonitoringConnectionType = MakeConnectionType( func(agentConfig *config.AgentConfig) bool { return agentConfig.SelfMonitoring.Enabled }, - []CollectorConfigFragment{ + []BundledConfigFragment{ { enabledCheck: func(agentConfig *config.AgentConfig) bool { return agentConfig.SelfMonitoring.Enabled diff --git a/internal/connections/bundledconfig/bundledconfig.go b/internal/connections/bundledconfig/bundledconfig.go new file mode 100644 index 000000000..044efc180 --- /dev/null +++ b/internal/connections/bundledconfig/bundledconfig.go @@ -0,0 +1,46 @@ +package bundledconfig + +import ( + "embed" + + "github.com/observeinc/observe-agent/internal/connections/bundledconfig/docker" + "github.com/observeinc/observe-agent/internal/connections/bundledconfig/linux" + "github.com/observeinc/observe-agent/internal/connections/bundledconfig/shared" + "github.com/observeinc/observe-agent/internal/connections/bundledconfig/windows" +) + +// TODO break up some of the larger connections in order to share more configs. +var SharedTemplateFS = map[string]embed.FS{ + "common/attributes.yaml.tmpl": shared.AttributesTemplateFS, + "common/internal_telemetry.yaml.tmpl": shared.InternalTelemetryTemplateFS, + "common/health_check.yaml.tmpl": shared.HealthCheckTemplateFS, + "common/base.yaml.tmpl": shared.BaseTemplateFS, + "common/forward.yaml.tmpl": shared.ForwardTemplateFS, + "host_monitoring/logs.yaml.tmpl": shared.LogsTemplateFS, + "host_monitoring/host_metrics.yaml.tmpl": shared.HostMetricsTemplateFS, + "host_monitoring/host.yaml.tmpl": shared.HostTemplateFS, + "host_monitoring/process_metrics.yaml.tmpl": shared.ProcessMetricsTemplateFS, + "self_monitoring/logs_and_metrics.yaml.tmpl": shared.LogsAndMetricsTemplateFS, +} + +var DockerTemplateFS = map[string]embed.FS{ + "common/base.yaml.tmpl": docker.BaseTemplateFS, + "host_monitoring/logs.yaml.tmpl": docker.LogsTemplateFS, + "host_monitoring/host_metrics.yaml.tmpl": docker.HostMetricsTemplateFS, + "host_monitoring/process_metrics.yaml.tmpl": docker.ProcessMetricsTemplateFS, + "self_monitoring/logs_and_metrics.yaml.tmpl": docker.LogsAndMetricsTemplateFS, +} + +var LinuxTemplateFS = map[string]embed.FS{ + "host_monitoring/logs.yaml.tmpl": linux.LogsTemplateFS, + "host_monitoring/host_metrics.yaml.tmpl": linux.HostMetricsTemplateFS, + "self_monitoring/logs_and_metrics.yaml.tmpl": linux.LogsAndMetricsTemplateFS, +} + +var MacOSTemplateFS = map[string]embed.FS{} + +var WindowsTemplateFS = map[string]embed.FS{ + "common/base.yaml.tmpl": windows.BaseTemplateFS, + "host_monitoring/logs.yaml.tmpl": windows.LogsTemplateFS, + "host_monitoring/host_metrics.yaml.tmpl": windows.HostMetricsTemplateFS, +} diff --git a/packaging/docker/observe-agent/connections/common/base.yaml.tmpl b/internal/connections/bundledconfig/docker/common/base.yaml.tmpl similarity index 97% rename from packaging/docker/observe-agent/connections/common/base.yaml.tmpl rename to internal/connections/bundledconfig/docker/common/base.yaml.tmpl index 751a74845..8dffa3960 100644 --- a/packaging/docker/observe-agent/connections/common/base.yaml.tmpl +++ b/internal/connections/bundledconfig/docker/common/base.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to cgroup extensions */ -}} extensions: file_storage: directory: ${env:FILESTORAGE_PATH} diff --git a/packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml.tmpl b/internal/connections/bundledconfig/docker/host_monitoring/host_metrics.yaml.tmpl similarity index 89% rename from packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml.tmpl rename to internal/connections/bundledconfig/docker/host_monitoring/host_metrics.yaml.tmpl index e669f0ddd..97afd2d76 100644 --- a/packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/docker/host_monitoring/host_metrics.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to hostfs root path */ -}} receivers: hostmetrics/host-monitoring-host: collection_interval: 60s @@ -26,6 +27,7 @@ receivers: system.filesystem.utilization: enabled: true network: + {{- /* TODO: add system.network.conntrack metrics, copy from the shared file */}} paging: metrics: system.paging.utilization: diff --git a/packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl b/internal/connections/bundledconfig/docker/host_monitoring/logs.yaml.tmpl similarity index 96% rename from packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl rename to internal/connections/bundledconfig/docker/host_monitoring/logs.yaml.tmpl index 4323b09fc..c748b4885 100644 --- a/packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl +++ b/internal/connections/bundledconfig/docker/host_monitoring/logs.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to hostfs root path and journald receiver */ -}} receivers: filelog/host_monitoring: include: diff --git a/packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml.tmpl b/internal/connections/bundledconfig/docker/host_monitoring/process_metrics.yaml.tmpl similarity index 95% rename from packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml.tmpl rename to internal/connections/bundledconfig/docker/host_monitoring/process_metrics.yaml.tmpl index 64c3c2818..99bfb88fa 100644 --- a/packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/docker/host_monitoring/process_metrics.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to hostfs root path */ -}} receivers: hostmetrics/host-monitoring-process: collection_interval: 60s diff --git a/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/internal/connections/bundledconfig/docker/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 91% rename from packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl rename to internal/connections/bundledconfig/docker/self_monitoring/logs_and_metrics.yaml.tmpl index 031b6b972..c395add84 100644 --- a/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/docker/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,8 @@ +{{- /* Forked from the default due to journald receiver */ -}} receivers: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields + {{- /* TODO update this to pull from the internal telemetry config */}} prometheus/agent: config: scrape_configs: @@ -37,6 +39,11 @@ service: - batch exporters: [prometheusremotewrite/observe] + # Deprecated + logs/agent-config: + receivers: [nop] + exporters: [nop] + logs/agent-journald: receivers: [journald/agent] processors: @@ -52,8 +59,3 @@ service: {{- end }} - batch exporters: [otlphttp/observe, count] - - # Deprecated - logs/agent-config: - receivers: [nop] - exporters: [nop] diff --git a/internal/connections/bundledconfig/docker/templates.go b/internal/connections/bundledconfig/docker/templates.go new file mode 100644 index 000000000..99c5e9b69 --- /dev/null +++ b/internal/connections/bundledconfig/docker/templates.go @@ -0,0 +1,16 @@ +package docker + +import "embed" + +var ( + //go:embed common/base.yaml.tmpl + BaseTemplateFS embed.FS + //go:embed host_monitoring/logs.yaml.tmpl + LogsTemplateFS embed.FS + //go:embed host_monitoring/host_metrics.yaml.tmpl + HostMetricsTemplateFS embed.FS + //go:embed host_monitoring/process_metrics.yaml.tmpl + ProcessMetricsTemplateFS embed.FS + //go:embed self_monitoring/logs_and_metrics.yaml.tmpl + LogsAndMetricsTemplateFS embed.FS +) diff --git a/internal/connections/bundledconfig/docker_defaults.go b/internal/connections/bundledconfig/docker_defaults.go new file mode 100644 index 000000000..59d11ca1a --- /dev/null +++ b/internal/connections/bundledconfig/docker_defaults.go @@ -0,0 +1,9 @@ +//go:build linux && docker + +package bundledconfig + +import ( + "embed" +) + +var OverrideTemplates map[string]embed.FS = DockerTemplateFS diff --git a/packaging/linux/connections/host_monitoring/host_metrics.yaml.tmpl b/internal/connections/bundledconfig/linux/host_monitoring/host_metrics.yaml.tmpl similarity index 88% rename from packaging/linux/connections/host_monitoring/host_metrics.yaml.tmpl rename to internal/connections/bundledconfig/linux/host_monitoring/host_metrics.yaml.tmpl index 53e2541ce..aff83f46b 100644 --- a/packaging/linux/connections/host_monitoring/host_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/linux/host_monitoring/host_metrics.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to system.linux.memory.available metric */ -}} receivers: hostmetrics/host-monitoring-host: collection_interval: 60s @@ -25,6 +26,7 @@ receivers: system.filesystem.utilization: enabled: true network: + {{- /* TODO: add system.network.conntrack metrics, copy from the shared file */}} paging: metrics: system.paging.utilization: diff --git a/packaging/linux/connections/host_monitoring/logs.yaml.tmpl b/internal/connections/bundledconfig/linux/host_monitoring/logs.yaml.tmpl similarity index 97% rename from packaging/linux/connections/host_monitoring/logs.yaml.tmpl rename to internal/connections/bundledconfig/linux/host_monitoring/logs.yaml.tmpl index ae33c1903..914e23673 100644 --- a/packaging/linux/connections/host_monitoring/logs.yaml.tmpl +++ b/internal/connections/bundledconfig/linux/host_monitoring/logs.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to journald receiver */ -}} receivers: filelog/host_monitoring: include: diff --git a/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/internal/connections/bundledconfig/linux/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 91% rename from packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl rename to internal/connections/bundledconfig/linux/self_monitoring/logs_and_metrics.yaml.tmpl index 031b6b972..c395add84 100644 --- a/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/linux/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,8 @@ +{{- /* Forked from the default due to journald receiver */ -}} receivers: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields + {{- /* TODO update this to pull from the internal telemetry config */}} prometheus/agent: config: scrape_configs: @@ -37,6 +39,11 @@ service: - batch exporters: [prometheusremotewrite/observe] + # Deprecated + logs/agent-config: + receivers: [nop] + exporters: [nop] + logs/agent-journald: receivers: [journald/agent] processors: @@ -52,8 +59,3 @@ service: {{- end }} - batch exporters: [otlphttp/observe, count] - - # Deprecated - logs/agent-config: - receivers: [nop] - exporters: [nop] diff --git a/internal/connections/bundledconfig/linux/templates.go b/internal/connections/bundledconfig/linux/templates.go new file mode 100644 index 000000000..5af0fd59d --- /dev/null +++ b/internal/connections/bundledconfig/linux/templates.go @@ -0,0 +1,12 @@ +package linux + +import "embed" + +var ( + //go:embed host_monitoring/logs.yaml.tmpl + LogsTemplateFS embed.FS + //go:embed host_monitoring/host_metrics.yaml.tmpl + HostMetricsTemplateFS embed.FS + //go:embed self_monitoring/logs_and_metrics.yaml.tmpl + LogsAndMetricsTemplateFS embed.FS +) diff --git a/internal/connections/bundledconfig/linux_defaults.go b/internal/connections/bundledconfig/linux_defaults.go new file mode 100644 index 000000000..5175c6023 --- /dev/null +++ b/internal/connections/bundledconfig/linux_defaults.go @@ -0,0 +1,9 @@ +//go:build linux && !docker + +package bundledconfig + +import ( + "embed" +) + +var OverrideTemplates map[string]embed.FS = LinuxTemplateFS diff --git a/internal/connections/bundledconfig/mac_defaults.go b/internal/connections/bundledconfig/mac_defaults.go new file mode 100644 index 000000000..493dec5a8 --- /dev/null +++ b/internal/connections/bundledconfig/mac_defaults.go @@ -0,0 +1,9 @@ +//go:build darwin + +package bundledconfig + +import ( + "embed" +) + +var OverrideTemplates map[string]embed.FS = MacOSTemplateFS diff --git a/internal/connections/bundledconfig/macos/templates.go b/internal/connections/bundledconfig/macos/templates.go new file mode 100644 index 000000000..3c9c360a9 --- /dev/null +++ b/internal/connections/bundledconfig/macos/templates.go @@ -0,0 +1,3 @@ +package macos + +// Add Mac specific templates here if we ever need them diff --git a/packaging/docker/observe-agent/connections/common/attributes.yaml.tmpl b/internal/connections/bundledconfig/shared/common/attributes.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/common/attributes.yaml.tmpl rename to internal/connections/bundledconfig/shared/common/attributes.yaml.tmpl diff --git a/packaging/macos/connections/common/base.yaml.tmpl b/internal/connections/bundledconfig/shared/common/base.yaml.tmpl similarity index 97% rename from packaging/macos/connections/common/base.yaml.tmpl rename to internal/connections/bundledconfig/shared/common/base.yaml.tmpl index 44a0cccbb..44d766410 100644 --- a/packaging/macos/connections/common/base.yaml.tmpl +++ b/internal/connections/bundledconfig/shared/common/base.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* TODO: split out extensions so this base file is more reusable */}} extensions: file_storage: directory: ${env:FILESTORAGE_PATH} diff --git a/packaging/docker/observe-agent/connections/common/forward.yaml.tmpl b/internal/connections/bundledconfig/shared/common/forward.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/common/forward.yaml.tmpl rename to internal/connections/bundledconfig/shared/common/forward.yaml.tmpl diff --git a/packaging/docker/observe-agent/connections/common/health_check.yaml.tmpl b/internal/connections/bundledconfig/shared/common/health_check.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/common/health_check.yaml.tmpl rename to internal/connections/bundledconfig/shared/common/health_check.yaml.tmpl diff --git a/packaging/docker/observe-agent/connections/common/internal_telemetry.yaml.tmpl b/internal/connections/bundledconfig/shared/common/internal_telemetry.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/common/internal_telemetry.yaml.tmpl rename to internal/connections/bundledconfig/shared/common/internal_telemetry.yaml.tmpl diff --git a/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl b/internal/connections/bundledconfig/shared/host_monitoring/host.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl rename to internal/connections/bundledconfig/shared/host_monitoring/host.yaml.tmpl diff --git a/packaging/macos/connections/host_monitoring/host_metrics.yaml.tmpl b/internal/connections/bundledconfig/shared/host_monitoring/host_metrics.yaml.tmpl similarity index 100% rename from packaging/macos/connections/host_monitoring/host_metrics.yaml.tmpl rename to internal/connections/bundledconfig/shared/host_monitoring/host_metrics.yaml.tmpl diff --git a/packaging/macos/connections/host_monitoring/logs.yaml.tmpl b/internal/connections/bundledconfig/shared/host_monitoring/logs.yaml.tmpl similarity index 100% rename from packaging/macos/connections/host_monitoring/logs.yaml.tmpl rename to internal/connections/bundledconfig/shared/host_monitoring/logs.yaml.tmpl diff --git a/packaging/linux/connections/host_monitoring/process_metrics.yaml.tmpl b/internal/connections/bundledconfig/shared/host_monitoring/process_metrics.yaml.tmpl similarity index 100% rename from packaging/linux/connections/host_monitoring/process_metrics.yaml.tmpl rename to internal/connections/bundledconfig/shared/host_monitoring/process_metrics.yaml.tmpl diff --git a/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/internal/connections/bundledconfig/shared/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 93% rename from packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl rename to internal/connections/bundledconfig/shared/self_monitoring/logs_and_metrics.yaml.tmpl index e45c0aae1..38d5fdf2e 100644 --- a/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/shared/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,7 @@ receivers: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields + {{- /* TODO update this to pull from the internal telemetry config */}} prometheus/agent: config: scrape_configs: diff --git a/internal/connections/bundledconfig/shared/templates.go b/internal/connections/bundledconfig/shared/templates.go new file mode 100644 index 000000000..9e41cabd1 --- /dev/null +++ b/internal/connections/bundledconfig/shared/templates.go @@ -0,0 +1,26 @@ +package shared + +import "embed" + +var ( + //go:embed common/attributes.yaml.tmpl + AttributesTemplateFS embed.FS + //go:embed common/internal_telemetry.yaml.tmpl + InternalTelemetryTemplateFS embed.FS + //go:embed common/health_check.yaml.tmpl + HealthCheckTemplateFS embed.FS + //go:embed common/base.yaml.tmpl + BaseTemplateFS embed.FS + //go:embed common/forward.yaml.tmpl + ForwardTemplateFS embed.FS + //go:embed host_monitoring/logs.yaml.tmpl + LogsTemplateFS embed.FS + //go:embed host_monitoring/host_metrics.yaml.tmpl + HostMetricsTemplateFS embed.FS + //go:embed host_monitoring/host.yaml.tmpl + HostTemplateFS embed.FS + //go:embed host_monitoring/process_metrics.yaml.tmpl + ProcessMetricsTemplateFS embed.FS + //go:embed self_monitoring/logs_and_metrics.yaml.tmpl + LogsAndMetricsTemplateFS embed.FS +) diff --git a/packaging/windows/connections/common/base.yaml.tmpl b/internal/connections/bundledconfig/windows/common/base.yaml.tmpl similarity index 96% rename from packaging/windows/connections/common/base.yaml.tmpl rename to internal/connections/bundledconfig/windows/common/base.yaml.tmpl index 54d56bc07..22e98e3b3 100644 --- a/packaging/windows/connections/common/base.yaml.tmpl +++ b/internal/connections/bundledconfig/windows/common/base.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to different resourcedetection settings */ -}} extensions: file_storage: directory: ${env:FILESTORAGE_PATH} diff --git a/packaging/windows/connections/host_monitoring/host_metrics.yaml.tmpl b/internal/connections/bundledconfig/windows/host_monitoring/host_metrics.yaml.tmpl similarity index 91% rename from packaging/windows/connections/host_monitoring/host_metrics.yaml.tmpl rename to internal/connections/bundledconfig/windows/host_monitoring/host_metrics.yaml.tmpl index 60f4dddd4..a3b38b22e 100644 --- a/packaging/windows/connections/host_monitoring/host_metrics.yaml.tmpl +++ b/internal/connections/bundledconfig/windows/host_monitoring/host_metrics.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to mismatch in collected metrics; TODO unify this */ -}} receivers: hostmetrics/host-monitoring-host: collection_interval: 60s diff --git a/packaging/windows/connections/host_monitoring/logs.yaml.tmpl b/internal/connections/bundledconfig/windows/host_monitoring/logs.yaml.tmpl similarity index 97% rename from packaging/windows/connections/host_monitoring/logs.yaml.tmpl rename to internal/connections/bundledconfig/windows/host_monitoring/logs.yaml.tmpl index eab3f07b0..b3786141b 100644 --- a/packaging/windows/connections/host_monitoring/logs.yaml.tmpl +++ b/internal/connections/bundledconfig/windows/host_monitoring/logs.yaml.tmpl @@ -1,3 +1,4 @@ +{{- /* Forked from the default due to windowseventlog receiver */ -}} receivers: windowseventlog/host_monitoring-application: channel: Application diff --git a/internal/connections/bundledconfig/windows/templates.go b/internal/connections/bundledconfig/windows/templates.go new file mode 100644 index 000000000..193edef62 --- /dev/null +++ b/internal/connections/bundledconfig/windows/templates.go @@ -0,0 +1,12 @@ +package windows + +import "embed" + +var ( + //go:embed common/base.yaml.tmpl + BaseTemplateFS embed.FS + //go:embed host_monitoring/logs.yaml.tmpl + LogsTemplateFS embed.FS + //go:embed host_monitoring/host_metrics.yaml.tmpl + HostMetricsTemplateFS embed.FS +) diff --git a/internal/connections/bundledconfig/windows_defaults.go b/internal/connections/bundledconfig/windows_defaults.go new file mode 100644 index 000000000..023edcece --- /dev/null +++ b/internal/connections/bundledconfig/windows_defaults.go @@ -0,0 +1,9 @@ +//go:build windows + +package bundledconfig + +import ( + "embed" +) + +var OverrideTemplates map[string]embed.FS = WindowsTemplateFS diff --git a/internal/connections/confighandler.go b/internal/connections/confighandler.go index da74bd2d9..25fec6b53 100644 --- a/internal/connections/confighandler.go +++ b/internal/connections/confighandler.go @@ -18,7 +18,7 @@ const ( ) func SetupAndGetConfigFiles(ctx context.Context) ([]string, func(), error) { - // Set up our temp dir annd temp config files + // Set up our temp dir and temp config files tmpDir, err := os.MkdirTemp("", TempFilesFolder) if err != nil { return nil, nil, err @@ -26,7 +26,7 @@ func SetupAndGetConfigFiles(ctx context.Context) ([]string, func(), error) { cleanup := func() { os.RemoveAll(tmpDir) } - configFilePaths, err := GetAllOtelConfigFilePaths(ctx, tmpDir) + configFilePaths, err := getAllOtelConfigFilePaths(ctx, tmpDir) if err != nil { cleanup() return nil, nil, err @@ -34,7 +34,7 @@ func SetupAndGetConfigFiles(ctx context.Context) ([]string, func(), error) { return configFilePaths, cleanup, nil } -func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, error) { +func getAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, error) { configFilePaths := []string{} // Get additional config paths based on connection configs agentConfig, err := config.AgentConfigFromViper(viper.GetViper()) @@ -42,7 +42,7 @@ func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, er return nil, err } for _, conn := range AllConnectionTypes { - connectionPaths, err := conn.GetConfigFilePaths(ctx, tmpDir, agentConfig) + connectionPaths, err := conn.GetBundledConfigs(ctx, tmpDir, agentConfig) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, er } // Only create the config file if there are overrides present (ie ignore empty maps) if len(overrides) != 0 { - overridePath, err := GetOverrideConfigFile(tmpDir, overrides) + overridePath, err := getOverrideConfigFile(tmpDir, overrides) if err != nil { return nil, err } @@ -77,7 +77,7 @@ func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, er return configFilePaths, nil } -func GetOverrideConfigFile(tmpDir string, data map[string]any) (string, error) { +func getOverrideConfigFile(tmpDir string, data map[string]any) (string, error) { f, err := os.CreateTemp(tmpDir, "otel-config-overrides-*.yaml") if err != nil { return "", fmt.Errorf("failed to create config file to write to: %w", err) diff --git a/internal/connections/connections.go b/internal/connections/connections.go index e338b18e7..51dc925bf 100644 --- a/internal/connections/connections.go +++ b/internal/connections/connections.go @@ -2,13 +2,16 @@ package connections import ( "context" + "embed" "fmt" "os" - "path/filepath" + "path" + "strings" "text/template" "github.com/observeinc/observe-agent/internal/commands/util/logger" "github.com/observeinc/observe-agent/internal/config" + "github.com/observeinc/observe-agent/internal/connections/bundledconfig" "go.uber.org/zap" ) @@ -16,59 +19,64 @@ var TempFilesFolder = "observe-agent" type EnabledCheckFn func(*config.AgentConfig) bool -type ConfigFieldHandler interface { - GenerateCollectorConfigFragment() interface{} -} - -type CollectorConfigFragment struct { +type BundledConfigFragment struct { enabledCheck EnabledCheckFn colConfigFilePath string } +type ConfigOverrides = map[string]embed.FS + type ConnectionType struct { - Name string - ConfigFields []CollectorConfigFragment - EnabledCheck EnabledCheckFn + Name string + BundledConfigFragments []BundledConfigFragment + EnabledCheck EnabledCheckFn - configFolderPath string + templateOverrides ConfigOverrides } -func (c *ConnectionType) GetTemplateFilepath(tplFilename string) string { - return filepath.Join(c.configFolderPath, c.Name, tplFilename) +func (c *ConnectionType) getTemplate(tplName string) (*template.Template, error) { + var fs embed.FS + var ok bool + if fs, ok = c.templateOverrides[tplName]; !ok { + fs, ok = bundledconfig.SharedTemplateFS[tplName] + if !ok { + return nil, fmt.Errorf("template %s not found", tplName) + } + } + return template.New(path.Base(tplName)).Funcs(TemplateFuncMap).ParseFS(fs, tplName) } -func RenderConfigTemplate(ctx context.Context, tmpDir string, tplPath string, confValues any) (string, error) { - _, tplFilename := filepath.Split(tplPath) - tmpl, err := template.New("").Funcs(GetTemplateFuncMap()).ParseFiles(tplPath) +func renderBundledConfigTemplate(ctx context.Context, tmpDir string, outFileName string, tmpl *template.Template, confValues any) (string, error) { + f, err := os.CreateTemp(tmpDir, fmt.Sprintf("*-%s", outFileName)) if err != nil { - logger.FromCtx(ctx).Error("failed to parse config fragment template", zap.String("file", tplPath), zap.Error(err)) + logger.FromCtx(ctx).Error("failed to create temporary config fragment file", zap.String("fileName", outFileName), zap.Error(err)) return "", err } - f, err := os.CreateTemp(tmpDir, fmt.Sprintf("*-%s", tplFilename)) + err = tmpl.Execute(f, confValues) if err != nil { - logger.FromCtx(ctx).Error("failed to create temporary config fragment file", zap.String("file", tplPath), zap.Error(err)) - return "", err - } - err = tmpl.ExecuteTemplate(f, tplFilename, confValues) - if err != nil { - logger.FromCtx(ctx).Error("failed to execute config fragment template", zap.String("file", tplPath), zap.Error(err)) + logger.FromCtx(ctx).Error("failed to execute config fragment template", zap.String("fileName", outFileName), zap.Error(err)) return "", err } return f.Name(), nil } -func (c *ConnectionType) RenderConfigTemplate(ctx context.Context, tmpDir string, tplFilename string, confValues any) (string, error) { - tplPath := c.GetTemplateFilepath(tplFilename) - return RenderConfigTemplate(ctx, tmpDir, tplPath, confValues) +func (c *ConnectionType) renderBundledConfigTemplate(ctx context.Context, tmpDir string, tplName string, confValues any) (string, error) { + tmpl, err := c.getTemplate(c.Name + "/" + tplName) + if err != nil { + fmt.Printf("TODO err1: %s\n", err.Error()) + return "", err + } + outFileName := c.Name + "-" + strings.TrimSuffix(tplName, ".tmpl") + return renderBundledConfigTemplate(ctx, tmpDir, outFileName, tmpl, confValues) } -func (c *ConnectionType) ProcessConfigFields(ctx context.Context, tmpDir string, agentConfig *config.AgentConfig) ([]string, error) { +func (c *ConnectionType) renderAllBundledConfigFragments(ctx context.Context, tmpDir string, agentConfig *config.AgentConfig) ([]string, error) { paths := make([]string, 0) - for _, field := range c.ConfigFields { - if !field.enabledCheck(agentConfig) || field.colConfigFilePath == "" { + for _, fragment := range c.BundledConfigFragments { + if !fragment.enabledCheck(agentConfig) || fragment.colConfigFilePath == "" { continue } - configPath, err := c.RenderConfigTemplate(ctx, tmpDir, field.colConfigFilePath, agentConfig) + configPath, err := c.renderBundledConfigTemplate(ctx, tmpDir, fragment.colConfigFilePath, agentConfig) if err != nil { return nil, err } @@ -77,12 +85,12 @@ func (c *ConnectionType) ProcessConfigFields(ctx context.Context, tmpDir string, return paths, nil } -func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string, agentConfig *config.AgentConfig) ([]string, error) { +func (c *ConnectionType) GetBundledConfigs(ctx context.Context, tmpDir string, agentConfig *config.AgentConfig) ([]string, error) { if !c.EnabledCheck(agentConfig) { return []string{}, nil } - configPaths, err := c.ProcessConfigFields(ctx, tmpDir, agentConfig) + configPaths, err := c.renderAllBundledConfigFragments(ctx, tmpDir, agentConfig) if err != nil { return nil, err } @@ -91,9 +99,9 @@ func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string, type ConnectionTypeOption func(*ConnectionType) -func MakeConnectionType(name string, enabledCheck EnabledCheckFn, configFields []CollectorConfigFragment, opts ...ConnectionTypeOption) *ConnectionType { - var c = &ConnectionType{Name: name, EnabledCheck: enabledCheck, ConfigFields: configFields} - c.configFolderPath = GetConfigFragmentFolderPath() +func MakeConnectionType(name string, enabledCheck EnabledCheckFn, fragments []BundledConfigFragment, opts ...ConnectionTypeOption) *ConnectionType { + var c = &ConnectionType{Name: name, EnabledCheck: enabledCheck, BundledConfigFragments: fragments} + c.templateOverrides = bundledconfig.OverrideTemplates // Apply provided options for _, opt := range opts { @@ -103,9 +111,9 @@ func MakeConnectionType(name string, enabledCheck EnabledCheckFn, configFields [ return c } -func WithConfigFolderPath(configFolderPath string) ConnectionTypeOption { +func WithConfigTemplateOverrides(templateOverrides ConfigOverrides) ConnectionTypeOption { return func(c *ConnectionType) { - c.configFolderPath = configFolderPath + c.templateOverrides = templateOverrides } } diff --git a/internal/connections/connections_test.go b/internal/connections/connections_test.go index 1e9d8ecf9..8665ae86e 100644 --- a/internal/connections/connections_test.go +++ b/internal/connections/connections_test.go @@ -2,6 +2,7 @@ package connections import ( "context" + "embed" "os" "path" "path/filepath" @@ -14,6 +15,18 @@ import ( "github.com/stretchr/testify/suite" ) +var ( + //go:embed test/testConfig1.tpl + testConfig1FS embed.FS + //go:embed test/testHelloWorld.tpl + testHelloWorldFS embed.FS +) + +var TestTemplateOverrides = map[string]embed.FS{ + "test/testConfig1.tpl": testConfig1FS, + "test/testHelloWorld.tpl": testHelloWorldFS, +} + type ConnectionsTestSuite struct { suite.Suite tempDir string @@ -39,12 +52,13 @@ func (suite *ConnectionsTestSuite) TearDownSuite() { var alwaysEnabled EnabledCheckFn = func(_ *config.AgentConfig) bool { return true } -func (suite *ConnectionsTestSuite) MakeConnectionType(configFields []CollectorConfigFragment, enableCheck EnabledCheckFn) *ConnectionType { +func (suite *ConnectionsTestSuite) MakeConnectionType(configFields []BundledConfigFragment, enableCheck EnabledCheckFn) *ConnectionType { return MakeConnectionType( "test", enableCheck, configFields, - WithConfigFolderPath(suite.configFilesPath)) + WithConfigTemplateOverrides(TestTemplateOverrides), + ) } func TestConnectionsTestSuite(t *testing.T) { @@ -52,7 +66,7 @@ func TestConnectionsTestSuite(t *testing.T) { } func (suite *ConnectionsTestSuite) TestConnections_RenderConfigTemplate() { - ct := suite.MakeConnectionType([]CollectorConfigFragment{ + ct := suite.MakeConnectionType([]BundledConfigFragment{ {enabledCheck: alwaysEnabled, colConfigFilePath: "config1.tpl"}, }, alwaysEnabled) @@ -76,7 +90,7 @@ func (suite *ConnectionsTestSuite) TestConnections_RenderConfigTemplate() { C: []string{"test1", "test2", "test3"}, }, } - result, err := ct.RenderConfigTemplate(suite.ctx, suite.tempDir, "testHelloWorld.tpl", confValues) + result, err := ct.renderBundledConfigTemplate(suite.ctx, suite.tempDir, "testHelloWorld.tpl", confValues) suite.NoError(err) suite.NotEmpty(result) @@ -95,19 +109,19 @@ func (suite *ConnectionsTestSuite) TestConnectionType_ProcessConfigFields() { agentConfig.SelfMonitoring.Enabled = false agentConfig.Forwarding.Enabled = true - ct := suite.MakeConnectionType([]CollectorConfigFragment{ + ct := suite.MakeConnectionType([]BundledConfigFragment{ {enabledCheck: func(ac *config.AgentConfig) bool { return ac.HostMonitoring.Enabled }, colConfigFilePath: "testConfig1.tpl"}, {enabledCheck: func(ac *config.AgentConfig) bool { return ac.SelfMonitoring.Enabled }, colConfigFilePath: "testConfig2.tpl"}, {enabledCheck: func(ac *config.AgentConfig) bool { return ac.Forwarding.Enabled }, colConfigFilePath: ""}, }, alwaysEnabled) - paths, err := ct.ProcessConfigFields(suite.ctx, suite.tempDir, &agentConfig) + paths, err := ct.renderAllBundledConfigFragments(suite.ctx, suite.tempDir, &agentConfig) suite.NoError(err) suite.Len(paths, 1) tmpFile := paths[0] tmpConfName := tmpFile[strings.LastIndex(tmpFile, "-")+1:] - suite.Equal(ct.ConfigFields[0].colConfigFilePath, tmpConfName) + suite.Equal(ct.BundledConfigFragments[0].colConfigFilePath, tmpConfName) } func (suite *ConnectionsTestSuite) TestConnectionType_GetConfigFilePaths() { @@ -117,22 +131,22 @@ func (suite *ConnectionsTestSuite) TestConnectionType_GetConfigFilePaths() { agentConfig.SelfMonitoring.Enabled = false agentConfig.Forwarding.Enabled = true - ct := suite.MakeConnectionType([]CollectorConfigFragment{ + ct := suite.MakeConnectionType([]BundledConfigFragment{ {enabledCheck: func(ac *config.AgentConfig) bool { return ac.HostMonitoring.Enabled }, colConfigFilePath: "testConfig1.tpl"}, {enabledCheck: func(ac *config.AgentConfig) bool { return ac.SelfMonitoring.Enabled }, colConfigFilePath: "testConfig2.tpl"}, {enabledCheck: func(ac *config.AgentConfig) bool { return ac.Forwarding.Enabled }, colConfigFilePath: ""}, }, func(ac *config.AgentConfig) bool { return ac.Debug }) - paths, err := ct.GetConfigFilePaths(suite.ctx, suite.tempDir, &agentConfig) + paths, err := ct.GetBundledConfigs(suite.ctx, suite.tempDir, &agentConfig) suite.NoError(err) suite.Len(paths, 1) tmpFile := paths[0] tmpConfName := tmpFile[strings.LastIndex(tmpFile, "-")+1:] - suite.Equal(ct.ConfigFields[0].colConfigFilePath, tmpConfName) + suite.Equal(ct.BundledConfigFragments[0].colConfigFilePath, tmpConfName) // Does nothing if not enabled agentConfig.Debug = false - paths, err = ct.GetConfigFilePaths(suite.ctx, suite.tempDir, &agentConfig) + paths, err = ct.GetBundledConfigs(suite.ctx, suite.tempDir, &agentConfig) suite.NoError(err) suite.Len(paths, 0) } diff --git a/internal/connections/templatefuncs.go b/internal/connections/templatefuncs.go index c30e2f861..4af8ac31c 100644 --- a/internal/connections/templatefuncs.go +++ b/internal/connections/templatefuncs.go @@ -8,20 +8,18 @@ import ( "gopkg.in/yaml.v3" ) -func GetTemplateFuncMap() template.FuncMap { - return template.FuncMap{ - "inlineArrayInt": TplInlineArray[int], - "inlineArrayStr": TplInlineArray[string], - "valToYaml": TplValueToYaml, - "objToYaml": TplToYaml, - "add": func(values ...int) int { - sum := 0 - for _, i := range values { - sum += i - } - return sum - }, - } +var TemplateFuncMap = template.FuncMap{ + "inlineArrayInt": TplInlineArray[int], + "inlineArrayStr": TplInlineArray[string], + "valToYaml": TplValueToYaml, + "objToYaml": TplToYaml, + "add": func(values ...int) int { + sum := 0 + for _, i := range values { + sum += i + } + return sum + }, } func TplInlineArray[T any](arr []T) string { diff --git a/packaging/linux/connections/common/attributes.yaml.tmpl b/packaging/linux/connections/common/attributes.yaml.tmpl deleted file mode 100644 index cecd8a581..000000000 --- a/packaging/linux/connections/common/attributes.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -processors: - {{- if .HasAttributes }} - attributes/observe_global_attributes: - actions: - {{- range $key, $value := .Attributes }} - - key: {{ $key }} - value: {{ $value }} - action: insert - {{- end }} - {{- end }} - {{- if .HasResourceAttributes }} - resource/observe_global_resource_attributes: - attributes: - {{- range $key, $value := .ResourceAttributes }} - - key: {{ $key }} - value: {{ $value }} - action: insert - {{- end }} - {{- end }} diff --git a/packaging/linux/connections/common/base.yaml.tmpl b/packaging/linux/connections/common/base.yaml.tmpl deleted file mode 100644 index 44a0cccbb..000000000 --- a/packaging/linux/connections/common/base.yaml.tmpl +++ /dev/null @@ -1,116 +0,0 @@ -extensions: - file_storage: - directory: ${env:FILESTORAGE_PATH} - -connectors: - count: - -receivers: - nop: - -processors: - # Snowflake limit for identifiers: Regardless of whether an identifier is unquoted or double-quoted, the maximum number of characters allowed is 255 (including blank spaces). - # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#identifier-requirements - transform/truncate: - log_statements: - - context: log - statements: - - truncate_all(attributes, 4095) - - truncate_all(resource.attributes, 4095) - trace_statements: - - context: span - statements: - - truncate_all(attributes, 4095) - - truncate_all(resource.attributes, 4095) - - memory_limiter: - check_interval: 1s - limit_percentage: 80 - spike_limit_percentage: 20 - - batch: - timeout: 5s - - deltatocumulative: - - resourcedetection: - detectors: [env, system] - system: - hostname_sources: ["dns", "os"] - resource_attributes: - host.id: - enabled: false - os.type: - enabled: true - host.arch: - enabled: true - host.name: - enabled: true - host.cpu.vendor.id: - enabled: true - host.cpu.family: - enabled: true - host.cpu.model.id: - enabled: true - host.cpu.model.name: - enabled: true - host.cpu.stepping: - enabled: true - host.cpu.cache.l2.size: - enabled: true - os.description: - enabled: true - - resourcedetection/cloud: - detectors: - {{- if .CloudResourceDetectors }} - {{- " " }}{{ inlineArrayStr .CloudResourceDetectors }} - {{- else }} ["gcp", "ecs", "ec2", "azure"] - {{- end }} - timeout: 2s - override: false - - filter/count: - error_mode: ignore - metrics: - metric: - - 'IsMatch(name, ".*")' - -exporters: - otlphttp/observe: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Host Explorer" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - - prometheusremotewrite/observe: - endpoint: ${env:OBSERVE_PROMETHEUS_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Host Explorer" - remote_write_queue: - num_consumers: 5 - max_batch_request_parallelism: 5 - resource_to_telemetry_conversion: - enabled: true # Convert resource attributes to metric labels - send_metadata: true - - debug: - nop: - -service: - pipelines: - metrics/count-nop-in: - receivers: [nop] - exporters: [count] - metrics/count-nop-out: - receivers: [count] - exporters: [nop] - - extensions: [health_check, file_storage] diff --git a/packaging/linux/connections/common/forward.yaml.tmpl b/packaging/linux/connections/common/forward.yaml.tmpl deleted file mode 100644 index 7feb88f89..000000000 --- a/packaging/linux/connections/common/forward.yaml.tmpl +++ /dev/null @@ -1,89 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - -exporters: - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - - otlphttp/observemetrics: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Metrics" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - -service: - pipelines: - metrics/forward: - receivers: [otlp] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - {{- if not .Forwarding.Metrics.OtlpMetrics }} - - deltatocumulative - {{- end }} - - batch - exporters: - {{- if .Forwarding.Metrics.OtlpMetrics }} - - otlphttp/observemetrics - {{- else }} - - prometheusremotewrite/observe - {{- end }} - - logs/forward: - receivers: [otlp] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observetracing] diff --git a/packaging/linux/connections/common/health_check.yaml.tmpl b/packaging/linux/connections/common/health_check.yaml.tmpl deleted file mode 100644 index dcd8a4a32..000000000 --- a/packaging/linux/connections/common/health_check.yaml.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -extensions: - health_check: - endpoint: {{ .HealthCheck.Endpoint }} - path: {{ .HealthCheck.Path }} diff --git a/packaging/linux/connections/common/internal_telemetry.yaml.tmpl b/packaging/linux/connections/common/internal_telemetry.yaml.tmpl deleted file mode 100644 index f7e98c64d..000000000 --- a/packaging/linux/connections/common/internal_telemetry.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -service: - telemetry: - {{- if .InternalTelemetry.Metrics.Enabled }} - metrics: - level: {{ .InternalTelemetry.Metrics.Level }} - readers: - - pull: - exporter: - prometheus: - host: {{ .InternalTelemetry.Metrics.Host }} - port: {{ .InternalTelemetry.Metrics.Port }} - {{- end }} - {{- if .InternalTelemetry.Logs.Enabled }} - logs: - level: {{ .InternalTelemetry.Logs.Level }} - {{- end }} diff --git a/packaging/linux/connections/host_monitoring/host.yaml.tmpl b/packaging/linux/connections/host_monitoring/host.yaml.tmpl deleted file mode 100644 index b45bce46c..000000000 --- a/packaging/linux/connections/host_monitoring/host.yaml.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -receivers: - filestats/agent: - include: "${env:OBSERVE_AGENT_CONFIG_PATH}" - collection_interval: 5m - initial_delay: 60s - -service: - pipelines: - metrics/agent-filestats: - receivers: [filestats/agent] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [prometheusremotewrite/observe] diff --git a/packaging/macos/connections/common/attributes.yaml.tmpl b/packaging/macos/connections/common/attributes.yaml.tmpl deleted file mode 100644 index cecd8a581..000000000 --- a/packaging/macos/connections/common/attributes.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -processors: - {{- if .HasAttributes }} - attributes/observe_global_attributes: - actions: - {{- range $key, $value := .Attributes }} - - key: {{ $key }} - value: {{ $value }} - action: insert - {{- end }} - {{- end }} - {{- if .HasResourceAttributes }} - resource/observe_global_resource_attributes: - attributes: - {{- range $key, $value := .ResourceAttributes }} - - key: {{ $key }} - value: {{ $value }} - action: insert - {{- end }} - {{- end }} diff --git a/packaging/macos/connections/common/forward.yaml.tmpl b/packaging/macos/connections/common/forward.yaml.tmpl deleted file mode 100644 index 7feb88f89..000000000 --- a/packaging/macos/connections/common/forward.yaml.tmpl +++ /dev/null @@ -1,89 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - -exporters: - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - - otlphttp/observemetrics: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Metrics" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - -service: - pipelines: - metrics/forward: - receivers: [otlp] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - {{- if not .Forwarding.Metrics.OtlpMetrics }} - - deltatocumulative - {{- end }} - - batch - exporters: - {{- if .Forwarding.Metrics.OtlpMetrics }} - - otlphttp/observemetrics - {{- else }} - - prometheusremotewrite/observe - {{- end }} - - logs/forward: - receivers: [otlp] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observetracing] diff --git a/packaging/macos/connections/common/health_check.yaml.tmpl b/packaging/macos/connections/common/health_check.yaml.tmpl deleted file mode 100644 index dcd8a4a32..000000000 --- a/packaging/macos/connections/common/health_check.yaml.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -extensions: - health_check: - endpoint: {{ .HealthCheck.Endpoint }} - path: {{ .HealthCheck.Path }} diff --git a/packaging/macos/connections/common/internal_telemetry.yaml.tmpl b/packaging/macos/connections/common/internal_telemetry.yaml.tmpl deleted file mode 100644 index f7e98c64d..000000000 --- a/packaging/macos/connections/common/internal_telemetry.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -service: - telemetry: - {{- if .InternalTelemetry.Metrics.Enabled }} - metrics: - level: {{ .InternalTelemetry.Metrics.Level }} - readers: - - pull: - exporter: - prometheus: - host: {{ .InternalTelemetry.Metrics.Host }} - port: {{ .InternalTelemetry.Metrics.Port }} - {{- end }} - {{- if .InternalTelemetry.Logs.Enabled }} - logs: - level: {{ .InternalTelemetry.Logs.Level }} - {{- end }} diff --git a/packaging/macos/connections/host_monitoring/host.yaml.tmpl b/packaging/macos/connections/host_monitoring/host.yaml.tmpl deleted file mode 100644 index b45bce46c..000000000 --- a/packaging/macos/connections/host_monitoring/host.yaml.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -receivers: - filestats/agent: - include: "${env:OBSERVE_AGENT_CONFIG_PATH}" - collection_interval: 5m - initial_delay: 60s - -service: - pipelines: - metrics/agent-filestats: - receivers: [filestats/agent] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [prometheusremotewrite/observe] diff --git a/packaging/macos/connections/host_monitoring/process_metrics.yaml.tmpl b/packaging/macos/connections/host_monitoring/process_metrics.yaml.tmpl deleted file mode 100644 index f8fec8d53..000000000 --- a/packaging/macos/connections/host_monitoring/process_metrics.yaml.tmpl +++ /dev/null @@ -1,43 +0,0 @@ -receivers: - hostmetrics/host-monitoring-process: - collection_interval: 60s - scrapers: - process: - metrics: - process.context_switches: - enabled: true - process.cpu.utilization: - enabled: true - process.disk.operations: - enabled: true - process.memory.utilization: - enabled: true - process.open_file_descriptors: - enabled: true - process.paging.faults: - enabled: true - process.signals_pending: - enabled: true - process.threads: - enabled: true - mute_process_name_error: true - mute_process_exe_error: true - mute_process_io_error: true - mute_process_user_error: true - -service: - pipelines: - metrics/host_monitoring_process: - receivers: [hostmetrics/host-monitoring-process] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [prometheusremotewrite/observe] diff --git a/packaging/windows/connections/common/attributes.yaml.tmpl b/packaging/windows/connections/common/attributes.yaml.tmpl deleted file mode 100644 index cecd8a581..000000000 --- a/packaging/windows/connections/common/attributes.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -processors: - {{- if .HasAttributes }} - attributes/observe_global_attributes: - actions: - {{- range $key, $value := .Attributes }} - - key: {{ $key }} - value: {{ $value }} - action: insert - {{- end }} - {{- end }} - {{- if .HasResourceAttributes }} - resource/observe_global_resource_attributes: - attributes: - {{- range $key, $value := .ResourceAttributes }} - - key: {{ $key }} - value: {{ $value }} - action: insert - {{- end }} - {{- end }} diff --git a/packaging/windows/connections/common/forward.yaml.tmpl b/packaging/windows/connections/common/forward.yaml.tmpl deleted file mode 100644 index 7feb88f89..000000000 --- a/packaging/windows/connections/common/forward.yaml.tmpl +++ /dev/null @@ -1,89 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - -exporters: - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - - otlphttp/observemetrics: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Metrics" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - -service: - pipelines: - metrics/forward: - receivers: [otlp] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - {{- if not .Forwarding.Metrics.OtlpMetrics }} - - deltatocumulative - {{- end }} - - batch - exporters: - {{- if .Forwarding.Metrics.OtlpMetrics }} - - otlphttp/observemetrics - {{- else }} - - prometheusremotewrite/observe - {{- end }} - - logs/forward: - receivers: [otlp] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [otlphttp/observetracing] diff --git a/packaging/windows/connections/common/health_check.yaml.tmpl b/packaging/windows/connections/common/health_check.yaml.tmpl deleted file mode 100644 index dcd8a4a32..000000000 --- a/packaging/windows/connections/common/health_check.yaml.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -extensions: - health_check: - endpoint: {{ .HealthCheck.Endpoint }} - path: {{ .HealthCheck.Path }} diff --git a/packaging/windows/connections/common/internal_telemetry.yaml.tmpl b/packaging/windows/connections/common/internal_telemetry.yaml.tmpl deleted file mode 100644 index f7e98c64d..000000000 --- a/packaging/windows/connections/common/internal_telemetry.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -service: - telemetry: - {{- if .InternalTelemetry.Metrics.Enabled }} - metrics: - level: {{ .InternalTelemetry.Metrics.Level }} - readers: - - pull: - exporter: - prometheus: - host: {{ .InternalTelemetry.Metrics.Host }} - port: {{ .InternalTelemetry.Metrics.Port }} - {{- end }} - {{- if .InternalTelemetry.Logs.Enabled }} - logs: - level: {{ .InternalTelemetry.Logs.Level }} - {{- end }} diff --git a/packaging/windows/connections/host_monitoring/host.yaml.tmpl b/packaging/windows/connections/host_monitoring/host.yaml.tmpl deleted file mode 100644 index b45bce46c..000000000 --- a/packaging/windows/connections/host_monitoring/host.yaml.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -receivers: - filestats/agent: - include: "${env:OBSERVE_AGENT_CONFIG_PATH}" - collection_interval: 5m - initial_delay: 60s - -service: - pipelines: - metrics/agent-filestats: - receivers: [filestats/agent] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [prometheusremotewrite/observe] diff --git a/packaging/windows/connections/host_monitoring/process_metrics.yaml.tmpl b/packaging/windows/connections/host_monitoring/process_metrics.yaml.tmpl deleted file mode 100644 index f8fec8d53..000000000 --- a/packaging/windows/connections/host_monitoring/process_metrics.yaml.tmpl +++ /dev/null @@ -1,43 +0,0 @@ -receivers: - hostmetrics/host-monitoring-process: - collection_interval: 60s - scrapers: - process: - metrics: - process.context_switches: - enabled: true - process.cpu.utilization: - enabled: true - process.disk.operations: - enabled: true - process.memory.utilization: - enabled: true - process.open_file_descriptors: - enabled: true - process.paging.faults: - enabled: true - process.signals_pending: - enabled: true - process.threads: - enabled: true - mute_process_name_error: true - mute_process_exe_error: true - mute_process_io_error: true - mute_process_user_error: true - -service: - pipelines: - metrics/host_monitoring_process: - receivers: [hostmetrics/host-monitoring-process] - processors: - - memory_limiter - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - batch - exporters: [prometheusremotewrite/observe] diff --git a/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl b/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl deleted file mode 100644 index e45c0aae1..000000000 --- a/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl +++ /dev/null @@ -1,38 +0,0 @@ -receivers: - # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - - prometheus/agent: - config: - scrape_configs: - - job_name: 'otelcol' - scrape_interval: 10s - static_configs: - - targets: ['0.0.0.0:8888'] - metric_relabel_configs: - - source_labels: [__name__] - regex: '.*grpc_io.*' - action: drop - -service: - pipelines: - metrics/agent-internal: - receivers: [prometheus/agent, count] - processors: - - memory_limiter - - transform/truncate - - resourcedetection - - resourcedetection/cloud - {{- if .HasAttributes }} - - attributes/observe_global_attributes - {{- end }} - {{- if .HasResourceAttributes }} - - resource/observe_global_resource_attributes - {{- end }} - - deltatocumulative - - batch - exporters: [prometheusremotewrite/observe] - - # Deprecated - logs/agent-config: - receivers: [nop] - exporters: [nop] diff --git a/scripts/install.ps1 b/scripts/install.ps1 index bc31c49bc..67a7e1947 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -24,7 +24,6 @@ $temp_dir="C:\temp" New-Item -ItemType Directory -Force -Path $temp_dir New-Item -ItemType Directory -Force -Path $observeagent_install_dir New-Item -ItemType Directory -Force -Path $observeagent_install_dir\config -New-Item -ItemType Directory -Force -Path $observeagent_install_dir\connections New-Item -ItemType Directory -Force -Path $program_data_filestorage if ($PSBoundParameters.ContainsKey('zip_dir')){ @@ -45,7 +44,10 @@ Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\observe-agent.exe if (Test-Path $temp_dir\observe-agent_Windows_x86_64\otel-collector.yaml) { Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\otel-collector.yaml -Destination $observeagent_install_dir\config\otel-collector.yaml } -Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\connections\* -Destination $observeagent_install_dir\connections -Recurse +if (Test-Path $temp_dir\observe-agent_Windows_x86_64\connections) { + New-Item -ItemType Directory -Force -Path $observeagent_install_dir\connections + Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\connections\* -Destination $observeagent_install_dir\connections -Recurse +} # If there's already an observe-agent.yaml we don't copy the template from the downloaded installation package and leave existing observe-agent.yaml alone if (-Not (Test-Path "$observeagent_install_dir\observe-agent.yaml")) diff --git a/scripts/install_linux.sh b/scripts/install_linux.sh index c789d568e..605dd3669 100755 --- a/scripts/install_linux.sh +++ b/scripts/install_linux.sh @@ -97,7 +97,9 @@ sudo rm -rf $observeagent_config_dir/otel-collector.yaml $observeagent_config_di if [ -f $tmp_dir/otel-collector.yaml ]; then sudo cp -f $tmp_dir/otel-collector.yaml $observeagent_config_dir/otel-collector.yaml fi -sudo cp -fR $tmp_dir/connections $observeagent_config_dir/connections +if [ -d $tmp_dir/connections ]; then + sudo cp -fR $tmp_dir/connections $observeagent_config_dir/connections +fi sudo chown -R root:root $observeagent_config_dir # Initialize the agent config file if it doesn't exist. diff --git a/scripts/install_mac.sh b/scripts/install_mac.sh index 8b2702d80..b81752ed6 100644 --- a/scripts/install_mac.sh +++ b/scripts/install_mac.sh @@ -104,7 +104,9 @@ sudo cp $tmp_dir/observe-agent $observeagent_install_dir/observe-agent if [ -d $tmp_dir/config ]; then sudo cp -R $tmp_dir/config $observeagent_install_dir/config fi -sudo cp -R $tmp_dir/connections $observeagent_install_dir/connections +if [ -d $tmp_dir/connections ]; then + sudo cp -R $tmp_dir/connections $observeagent_install_dir/connections +fi sudo chown -R root:wheel $observeagent_install_dir # Initialize the agent config file if it doesn't exist