refactor: split main otel config into multiple connections, add tests (#186)

### Description

OB-36266: Split main otel config into multiple connections. Use the
agent config object as the source of truth instead of viper. Add tests.

I also tested this with the new `observe-agent config --render-otel`
command. I enabled all options in my agent config, and diffed the output
between `main` and this branch and only these lines changed:
```diff
104c104
<             - /usr/local/observe-agent/config/otel-collector.yaml
---
>             - /usr/local/observe-agent/connections/common/base.yaml.tmpl
123c123
<         include: /usr/local/observe-agent/config/otel-collector.yaml
---
>         include: /usr/local/observe-agent/connections/common/base.yaml.tmpl
```
This is expected since the otel-collector.yaml file no longer exists.

### Checklist
- [x] Created tests which fail without the change (if possible)

Author: obs-gh-mattcotter

.github/workflows/generate-changelog.yaml

@@ -22,7 +22,7 @@ jobs:
           # either 'goreleaser' (default) or 'goreleaser-pro'
           distribution: goreleaser-pro
           # 'latest', 'nightly', or a semver
-          version: 2.7.0
+          version: 2.8.2
           args: changelog
         env:
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.github/workflows/release-build.yaml

@@ -32,7 +32,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser-pro
-          version: 2.7.0
+          version: 2.8.2
           args: release --prepare --clean --snapshot --verbose
         env:
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.github/workflows/release-nightly.yaml

@@ -65,7 +65,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser-pro
-          version: 2.7.0
+          version: 2.8.2
           args: release --clean --skip=validate --verbose --nightly --parallelism 6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -60,7 +60,7 @@ jobs:
         with:
           distribution: goreleaser-pro
           # 'latest', 'nightly', or a semver
-          version: 2.7.0
+          version: 2.8.2
           args: release --clean --skip=validate --verbose
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests-integration.yaml

@@ -41,7 +41,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser-pro
-          version: 2.7.0
+          version: 2.8.2
           args: release --prepare --clean --snapshot --verbose --parallelism 6
         env:
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.github/workflows/vuln-check-full.yaml

@@ -42,7 +42,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser-pro
-          version: 2.7.0
+          version: 2.8.2
           args: build --snapshot --id=default --skip=validate --single-target
         env:
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.github/workflows/vuln-check-release.yaml

@@ -41,7 +41,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser-pro
-          version: 2.7.0
+          version: 2.8.2
           args: build --snapshot --id=default --skip=validate --single-target
         env:
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.goreleaser.yaml

@@ -16,7 +16,7 @@ before:
     - go generate ./...
 
 builds:
-  - id: default
+  - id: linux_build
     binary: observe-agent
     env:
       - CGO_ENABLED=0
@@ -44,10 +44,9 @@ builds:
     ldflags: -s -w -X github.com/observeinc/observe-agent/build.Version={{.Version}}
 
 archives:
-  - id: default
+  - id: linux
     formats: ["tar.gz"]
-    builds:
-      - default
+    ids: [linux_build]
     # this name template makes the OS and Arch compatible with the results of `uname`.
     name_template: >-
       {{ .ProjectName }}_
@@ -57,15 +56,15 @@ archives:
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
     files:
-      - src: "packaging/linux/etc/observe-agent/*"
-        dst: "."
+      # The default agent config file and the service file live in config.
       - src: "packaging/linux/config/*"
         dst: "."
+      - src: "packaging/linux/connections/*"
+        dst: "connections"
 
   - id: windows
     formats: ["zip"]
-    builds:
-      - windows_build
+    ids: [windows_build]
     name_template: >-
       {{ .ProjectName }}_
       {{- title .Os }}_
@@ -74,19 +73,14 @@ archives:
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
     files:
-      - src: "packaging/windows/config/otel-collector.yaml"
-        dst: "otel-collector.yaml"
       - src: "packaging/windows/observe-agent.yaml"
         dst: "observe-agent.yaml"
-      - src: "packaging/windows/connections/host_monitoring/*"
-        dst: "connections/host_monitoring"
-      - src: "packaging/windows/connections/self_monitoring/*"
-        dst: "connections/self_monitoring"
+      - src: "packaging/windows/connections/*"
+        dst: "connections"
 
   - id: macos
     formats: ["zip"]
-    builds:
-      - mac_build
+    ids: [mac_build]
     name_template: >-
       {{ .ProjectName }}_
       {{- title .Os }}_
@@ -124,7 +118,7 @@ changelog:
       order: 999
 
 nfpms:
-  - id: default
+  - id: linux
 
     package_name: observe-agent
 
@@ -168,8 +162,8 @@ nfpms:
     # Contents to add to the package.
     # GoReleaser will automatically add the binaries.
     contents:
-      - src: packaging/linux/etc/
-        dst: /etc
+      - src: packaging/linux/connections
+        dst: /etc/observe-agent/connections
         type: tree
 
       - src: packaging/linux/config/observe-agent.yaml

go.mod

@@ -5,6 +5,7 @@ go 1.23.7
 require (
 	github.com/go-viper/mapstructure/v2 v2.2.1
 	github.com/jarcoal/httpmock v1.3.1
+	github.com/mcuadros/go-defaults v1.2.0
 	github.com/observeinc/observe-agent/observecol v0.0.0-00010101000000-000000000000
 	github.com/prometheus/client_model v0.6.1
 	github.com/prometheus/common v0.62.0

integration/scripts/install_windows.ps1

@@ -18,7 +18,7 @@ $temp_dir="C:\temp"
 #Create directories for temp & observe-agent installation ls
 New-Item -ItemType Directory -Force -Path $temp_dir
 New-Item -ItemType Directory -Force -Path $observeagent_install_dir
-New-Item -ItemType Directory -Force -Path $observeagent_install_dir\config
+New-Item -ItemType Directory -Force -Path $observeagent_install_dir\connections
 New-Item -ItemType Directory -Force -Path $program_data_filestorage
 
 # Stop the observe agent if its running so that we can copy the new .exe
@@ -36,7 +36,5 @@ Expand-Archive -Force -LiteralPath $local_installer  -DestinationPath "$temp_dir
 Write-Output "Copying files from $temp_dir\observe-agent_extract to $observeagent_install_dir"
 Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.exe -Destination $observeagent_install_dir
 Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.yaml -Destination $observeagent_install_dir
-Copy-Item -Force -Path $temp_dir\observe-agent_extract\otel-collector.yaml -Destination $observeagent_install_dir\config\otel-collector.yaml
-Copy-Item -Force -Path $temp_dir\observe-agent_extract\connections\ -Destination $observeagent_install_dir\connections -Recurse
-
-
+Copy-Item -Force -Path $temp_dir\observe-agent_extract\connections\* -Destination $observeagent_install_dir\connections -Recurse
+Get-ChildItem -Path $observeagent_install_dir -Recurse