fix: correctly handle viper env var overrides for slice fields

Author: obs-gh-mattcotter

internal/commands/config/config.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/observeinc/observe-agent/internal/commands/start"
 	logger "github.com/observeinc/observe-agent/internal/commands/util"
+	"github.com/observeinc/observe-agent/internal/config"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -30,19 +31,19 @@ OTEL configuration.`,
 		if cleanup != nil {
 			defer cleanup()
 		}
-		var viperConfig map[string]any
-		if err := viper.Unmarshal(&viperConfig); err != nil {
+		agentConfig, err := config.AgentConfigFromViper(viper.GetViper())
+		if err != nil {
 			return err
 		}
-		viperConfigYaml, err := yaml.Marshal(viperConfig)
+		agentConfigYaml, err := yaml.Marshal(agentConfig)
 		if err != nil {
 			return err
 		}
 		fmt.Printf("# ======== computed agent config\n")
-		fmt.Println(string(viperConfigYaml) + "\n")
-		agentConfig := viper.ConfigFileUsed()
-		if agentConfig != "" {
-			configFilePaths = append([]string{agentConfig}, configFilePaths...)
+		fmt.Println(string(agentConfigYaml) + "\n")
+		agentConfigFile := viper.ConfigFileUsed()
+		if agentConfigFile != "" {
+			configFilePaths = append([]string{agentConfigFile}, configFilePaths...)
 		}
 		for _, filePath := range configFilePaths {
 			file, err := os.ReadFile(filePath)

internal/config/configschema.go

@@ -7,64 +7,48 @@ import (
 	"strings"
 
 	"github.com/spf13/viper"
-	"gopkg.in/yaml.v3"
 )
 
 type HostMonitoringLogsConfig struct {
-	Enabled bool     `yaml:"enabled"`
-	Include []string `yaml:"include,omitempty"`
+	Enabled bool     `yaml:"enabled" mapstructure:"enabled"`
+	Include []string `yaml:"include,omitempty" mapstructure:"include"`
 }
 
 type HostMonitoringHostMetricsConfig struct {
-	Enabled bool `yaml:"enabled"`
+	Enabled bool `yaml:"enabled" mapstructure:"enabled"`
 }
 
 type HostMonitoringProcessMetricsConfig struct {
-	Enabled bool `yaml:"enabled"`
+	Enabled bool `yaml:"enabled" mapstructure:"enabled"`
 }
 
 type HostMonitoringMetricsConfig struct {
-	Host    HostMonitoringHostMetricsConfig    `yaml:"host,omitempty"`
-	Process HostMonitoringProcessMetricsConfig `yaml:"process,omitempty"`
+	Host    HostMonitoringHostMetricsConfig    `yaml:"host,omitempty" mapstructure:"host"`
+	Process HostMonitoringProcessMetricsConfig `yaml:"process,omitempty" mapstructure:"process"`
 }
 
 type HostMonitoringConfig struct {
-	Enabled bool                        `yaml:"enabled"`
-	Logs    HostMonitoringLogsConfig    `yaml:"logs,omitempty"`
-	Metrics HostMonitoringMetricsConfig `yaml:"metrics,omitempty"`
+	Enabled bool                        `yaml:"enabled" mapstructure:"enabled"`
+	Logs    HostMonitoringLogsConfig    `yaml:"logs,omitempty" mapstructure:"logs"`
+	Metrics HostMonitoringMetricsConfig `yaml:"metrics,omitempty" mapstructure:"metrics"`
 }
 
 type SelfMonitoringConfig struct {
-	Enabled bool `yaml:"enabled"`
+	Enabled bool `yaml:"enabled" mapstructure:"enabled"`
 }
 
 type AgentConfig struct {
-	Token                  string               `yaml:"token"`
-	ObserveURL             string               `yaml:"observe_url"`
-	CloudResourceDetectors []string             `yaml:"cloud_resource_detectors,omitempty"`
-	SelfMonitoring         SelfMonitoringConfig `yaml:"self_monitoring,omitempty"`
-	HostMonitoring         HostMonitoringConfig `yaml:"host_monitoring,omitempty"`
-	OtelConfigOverrides    map[string]any       `yaml:"otel_config_overrides,omitempty"`
-}
-
-func UnmarshalViperThroughYaml(v *viper.Viper, out any) error {
-	// First unmarshal viper into a map
-	var viperConfig map[string]any
-	if err := viper.Unmarshal(&viperConfig); err != nil {
-		return err
-	}
-	// Next convert the map into yaml bytes
-	viperConfigYaml, err := yaml.Marshal(viperConfig)
-	if err != nil {
-		return err
-	}
-	// Finally unmarshal the yaml bytes into the out struct
-	return yaml.Unmarshal(viperConfigYaml, out)
+	Token                  string               `yaml:"token" mapstructure:"token"`
+	ObserveURL             string               `yaml:"observe_url" mapstructure:"observe_url"`
+	CloudResourceDetectors []string             `yaml:"cloud_resource_detectors,omitempty" mapstructure:"cloud_resource_detectors"`
+	SelfMonitoring         SelfMonitoringConfig `yaml:"self_monitoring,omitempty" mapstructure:"self_monitoring"`
+	HostMonitoring         HostMonitoringConfig `yaml:"host_monitoring,omitempty" mapstructure:"host_monitoring"`
+	OtelConfigOverrides    map[string]any       `yaml:"otel_config_overrides,omitempty" mapstructure:"otel_config_overrides"`
 }
 
 func AgentConfigFromViper(v *viper.Viper) (*AgentConfig, error) {
 	var config AgentConfig
-	err := UnmarshalViperThroughYaml(v, &config)
+	err := v.Unmarshal(&config)
 	if err != nil {
 		return nil, err
 	}

internal/connections/connections.go

@@ -86,7 +86,7 @@ func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string)
 	switch c.Type {
 	case SelfMonitoringConnectionTypeName:
 		conf := &config.SelfMonitoringConfig{}
-		err := config.UnmarshalViperThroughYaml(rawConnConfig, conf)
+		err := rawConnConfig.Unmarshal(conf)
 		if err != nil {
 			logger.FromCtx(ctx).Error("failed to unmarshal config", zap.String("connection", c.Name))
 			return nil, err
@@ -97,7 +97,7 @@ func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string)
 		}
 	case HostMonitoringConnectionTypeName:
 		conf := &config.HostMonitoringConfig{}
-		err := config.UnmarshalViperThroughYaml(rawConnConfig, conf)
+		err := rawConnConfig.Unmarshal(conf)
 		if err != nil {
 			logger.FromCtx(ctx).Error("failed to unmarshal config", zap.String("connection", c.Name))
 			return nil, err

packaging/docker/observe-agent/connections/host_monitoring/logs.yaml

@@ -6,8 +6,10 @@ receivers:
       - {{ . }}
       {{- end }}
       {{- else }}
+      - /var/log/**/*.log
+      - /var/log/syslog
       - /hostfs/var/log/**/*.log
-      - /hostfs/var/log/syslog]
+      - /hostfs/var/log/syslog
       {{- end }}
     include_file_path: true
     storage: file_storage