feat: support otelcol flags (config, set, feature-gates) in all sub commands

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/prometheus/common v0.62.0
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/spf13/cobra v1.9.1
+	github.com/spf13/pflag v1.0.6
 	github.com/spf13/viper v1.20.0-alpha.6
 	github.com/stretchr/testify v1.10.0
 	go.opentelemetry.io/collector/confmap v1.30.0
@@ -293,7 +294,6 @@ require (
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tidwall/gjson v1.14.2 // indirect

internal/commands/config/config.go

@@ -6,9 +6,8 @@ package config
 import (
 	"context"
 	"fmt"
+	"os"
 
-	"github.com/observeinc/observe-agent/internal/commands/start"
-	"github.com/observeinc/observe-agent/internal/commands/util"
 	"github.com/observeinc/observe-agent/internal/commands/util/logger"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/spf13/cobra"
@@ -33,19 +32,12 @@ bundled OTel configuration.`,
 		}
 
 		ctx := logger.WithCtx(context.Background(), logger.GetNop())
-		configFilePaths, cleanup, err := start.SetupAndGetConfigFiles(ctx)
-		if cleanup != nil {
-			defer cleanup()
-		}
-		if err != nil {
-			return err
-		}
 		if singleOtel {
-			return util.PrintShortOtelConfig(ctx, configFilePaths)
+			return PrintShortOtelConfig(ctx, os.Stdout)
 		} else if detailedOtel {
-			return util.PrintFullOtelConfig(configFilePaths)
+			return PrintFullOtelConfig(ctx, os.Stdout)
 		}
-		return util.PrintAllConfigsIndividually(configFilePaths)
+		return PrintAllConfigsIndividually(ctx, os.Stdout)
 	},
 }
 

internal/commands/config/config_test.go

@@ -0,0 +1,52 @@
+/*
+Copyright © 2024 NAME HERE <EMAIL ADDRESS>
+*/
+package config
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/observeinc/observe-agent/internal/commands/util/logger"
+	"github.com/observeinc/observe-agent/internal/connections"
+	"github.com/observeinc/observe-agent/internal/root"
+	"github.com/observeinc/observe-agent/observecol"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+// TODO rework this test to handle our snapshot tests as go unit tests.
+func Test_RenderOtelConfig(t *testing.T) {
+	// Get current path
+	_, filename, _, ok := runtime.Caller(0)
+	assert.True(t, ok)
+	curPath := path.Dir(filename)
+
+	// Set the template base dir for all connections
+	for _, conn := range connections.AllConnectionTypes {
+		conn.ApplyOptions(connections.WithConfigFolderPath(filepath.Join(curPath, "../../../packaging/macos/connections")))
+	}
+
+	// Set config flags
+	flags := pflag.NewFlagSet("test", pflag.ContinueOnError)
+	observecol.AddConfigFlags(flags)
+	flags.Parse([]string{"--config", filepath.Join(curPath, "test/otel-config.yaml")})
+	viper.Reset()
+	root.CfgFile = filepath.Join(curPath, "test/agent-config.yaml")
+	root.InitConfig()
+
+	// Run the test
+	ctx := logger.WithCtx(context.Background(), logger.GetNop())
+	var output bytes.Buffer
+	PrintShortOtelConfig(ctx, &output)
+	expected, err := os.ReadFile(filepath.Join(curPath, "test/output.yaml"))
+	assert.NoError(t, err)
+	assert.Equal(t, strings.TrimSpace(string(expected)), strings.TrimSpace(output.String()))
+}

internal/commands/util/config_printers.go → internal/commands/config/printers.go

@@ -1,25 +1,35 @@
-package util
+package config
 
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
 	"github.com/go-viper/mapstructure/v2"
 	"github.com/observeinc/observe-agent/internal/config"
+	"github.com/observeinc/observe-agent/internal/connections"
 	"github.com/observeinc/observe-agent/observecol"
 	"github.com/spf13/viper"
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/otelcol"
 	"gopkg.in/yaml.v3"
 )
 
-func PrintAllConfigsIndividually(configFilePaths []string) error {
+func PrintAllConfigsIndividually(ctx context.Context, w io.Writer) error {
+	configFilePaths, cleanup, err := connections.SetupAndGetConfigFiles(ctx)
+	if cleanup != nil {
+		defer cleanup()
+	}
+	if err != nil {
+		return err
+	}
+
 	printConfig := func(comment string, data []byte) {
-		fmt.Printf("# ======== %s\n", comment)
-		fmt.Println(strings.Trim(string(data), "\n\t "))
-		fmt.Println("---")
+		fmt.Fprintf(w, "# ======== %s\n", comment)
+		fmt.Fprintln(w, strings.Trim(string(data), "\n\t "))
+		fmt.Fprintln(w, "---")
 	}
 
 	agentConfig, err := config.AgentConfigFromViper(viper.GetViper())
@@ -52,12 +62,18 @@ func PrintAllConfigsIndividually(configFilePaths []string) error {
 	return nil
 }
 
-func PrintShortOtelConfig(ctx context.Context, configFilePaths []string) error {
-	if len(configFilePaths) == 0 {
+func PrintShortOtelConfig(ctx context.Context, w io.Writer) error {
+	settings, cleanup, err := observecol.GetOtelCollectorSettings(ctx)
+	if cleanup != nil {
+		defer cleanup()
+	}
+	if err != nil {
+		return err
+	}
+	if len(settings.ConfigProviderSettings.ResolverSettings.URIs) == 0 {
 		return nil
 	}
-	settings := observecol.ConfigProviderSettings(configFilePaths)
-	resolver, err := confmap.NewResolver(settings.ResolverSettings)
+	resolver, err := confmap.NewResolver(settings.ConfigProviderSettings.ResolverSettings)
 	if err != nil {
 		return fmt.Errorf("failed to create new resolver: %w", err)
 	}
@@ -69,20 +85,26 @@ func PrintShortOtelConfig(ctx context.Context, configFilePaths []string) error {
 	if err != nil {
 		return fmt.Errorf("error while marshaling to YAML: %w", err)
 	}
-	fmt.Printf("%s\n", b)
+	fmt.Fprintf(w, "%s\n", b)
 	return nil
 }
 
-func PrintFullOtelConfig(configFilePaths []string) error {
-	if len(configFilePaths) == 0 {
+func PrintFullOtelConfig(ctx context.Context, w io.Writer) error {
+	settings, cleanup, err := observecol.GetOtelCollectorSettings(ctx)
+	if cleanup != nil {
+		defer cleanup()
+	}
+	if err != nil {
+		return err
+	}
+	if len(settings.ConfigProviderSettings.ResolverSettings.URIs) == 0 {
 		return nil
 	}
-	colSettings := observecol.GenerateCollectorSettingsWithConfigFiles(configFilePaths)
-	factories, err := colSettings.Factories()
+	factories, err := settings.Factories()
 	if err != nil {
 		return fmt.Errorf("failed to create component factory maps: %w", err)
 	}
-	provider, err := otelcol.NewConfigProvider(colSettings.ConfigProviderSettings)
+	provider, err := otelcol.NewConfigProvider(settings.ConfigProviderSettings)
 	if err != nil {
 		return fmt.Errorf("failed to create config provider: %w", err)
 	}
@@ -99,6 +121,6 @@ func PrintFullOtelConfig(configFilePaths []string) error {
 	if err != nil {
 		return fmt.Errorf("failed to marshall config to yaml: %w", err)
 	}
-	fmt.Printf("%s\n", cfgYaml)
+	fmt.Fprintf(w, "%s\n", cfgYaml)
 	return nil
 }

internal/commands/config/test/agent-config.yaml

@@ -0,0 +1,22 @@
+# Target Observe collection url
+observe_url: https://test.collect.observeinc.com/
+
+# Observe data token
+token: 12345678901234567890:abcdefghijklmnopqrstuvwxyzABCDEF
+
+debug: false
+
+forwarding:
+    enabled: false
+
+health_check:
+    enabled: false
+
+internal_telemetry:
+    enabled: false
+
+host_monitoring:
+    enabled: false
+
+self_monitoring:
+    enabled: false

internal/commands/config/test/otel-config.yaml

@@ -0,0 +1,17 @@
+receivers:
+  filelog/test:
+    include: ["./test.log"]
+
+service:
+    pipelines:
+        logs/test:
+            receivers:
+                - filelog/test
+            processors:
+                - memory_limiter
+                - transform/truncate
+                - resourcedetection
+                - resourcedetection/cloud
+                - batch
+            exporters:
+                - otlphttp/observe

internal/commands/config/test/output.yaml

@@ -0,0 +1,124 @@
+connectors:
+    count: null
+exporters:
+    debug: null
+    nop: null
+    otlphttp/observe:
+        compression: zstd
+        endpoint: https://test.collect.observeinc.com/v2/otel
+        headers:
+            authorization: Bearer 12345678901234567890:abcdefghijklmnopqrstuvwxyzABCDEF
+            x-observe-target-package: Host Explorer
+        retry_on_failure:
+            enabled: true
+        sending_queue:
+            num_consumers: 4
+            queue_size: 100
+    prometheusremotewrite/observe:
+        endpoint: https://test.collect.observeinc.com/v1/prometheus
+        headers:
+            authorization: Bearer 12345678901234567890:abcdefghijklmnopqrstuvwxyzABCDEF
+            x-observe-target-package: Host Explorer
+        max_batch_request_parallelism: 5
+        remote_write_queue:
+            num_consumers: 5
+        resource_to_telemetry_conversion:
+            enabled: true
+        send_metadata: true
+extensions:
+    file_storage:
+        directory: /var/lib/observe-agent/filestorage
+processors:
+    batch:
+        timeout: 5s
+    deltatocumulative: null
+    filter/count:
+        error_mode: ignore
+        metrics:
+            metric:
+                - IsMatch(name, ".*")
+    memory_limiter:
+        check_interval: 1s
+        limit_percentage: 80
+        spike_limit_percentage: 20
+    resourcedetection:
+        detectors:
+            - env
+            - system
+        system:
+            hostname_sources:
+                - dns
+                - os
+            resource_attributes:
+                host.arch:
+                    enabled: true
+                host.cpu.cache.l2.size:
+                    enabled: true
+                host.cpu.family:
+                    enabled: true
+                host.cpu.model.id:
+                    enabled: true
+                host.cpu.model.name:
+                    enabled: true
+                host.cpu.stepping:
+                    enabled: true
+                host.cpu.vendor.id:
+                    enabled: true
+                host.id:
+                    enabled: false
+                host.name:
+                    enabled: true
+                os.description:
+                    enabled: true
+                os.type:
+                    enabled: true
+    resourcedetection/cloud:
+        detectors:
+            - gcp
+            - ecs
+            - ec2
+            - azure
+        override: false
+        timeout: 2s
+    transform/truncate:
+        log_statements:
+            - context: log
+              statements:
+                - truncate_all(attributes, 4095)
+                - truncate_all(resource.attributes, 4095)
+        trace_statements:
+            - context: span
+              statements:
+                - truncate_all(attributes, 4095)
+                - truncate_all(resource.attributes, 4095)
+receivers:
+    filelog/test:
+        include:
+            - ./test.log
+    nop: null
+service:
+    extensions:
+        - health_check
+        - file_storage
+    pipelines:
+        logs/test:
+            exporters:
+                - otlphttp/observe
+            processors:
+                - memory_limiter
+                - transform/truncate
+                - resourcedetection
+                - resourcedetection/cloud
+                - batch
+            receivers:
+                - filelog/test
+        metrics/count-nop-in:
+            exporters:
+                - count
+            receivers:
+                - nop
+        metrics/count-nop-out:
+            exporters:
+                - nop
+            receivers:
+                - count