feat: use agent config for init-config command future proofing (#189)

### Description

Use agent config for `init-config` command. This allows all values in
the agent config to work with this command both currently and going
forward.

Ex:
```
$ OBSERVE_URL="http://observeinc.com/" ./observe-agent --observe-config=./empty.yaml init-config --print 2>/dev/null
# Observe data token
token: ""

# Target Observe collection url
observe_url: http://observeinc.com/

health_check:
    enabled: true
    endpoint: localhost:13133
    path: /status

forwarding:
    enabled: true

internal_telemetry:
    enabled: true
    host: localhost
    port: 8888

# otel_config_overrides:
#   exporters:
#     debug:
#       verbosity: detailed
#       sampling_initial: 5
#       sampling_thereafter: 200
#   service:
#     pipelines:
#       metrics:
#         receivers: [hostmetrics]
#         processors: [memory_limiter]
#         exporters: [debug]
```

Author: obs-gh-mattcotter

internal/commands/initconfig/initconfig.go

@@ -4,11 +4,10 @@ Copyright © 2024 NAME HERE <EMAIL ADDRESS>
 package initconfig
 
 import (
-	"embed"
 	"fmt"
-	"html/template"
 	"os"
 
+	"github.com/observeinc/observe-agent/internal/config"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -25,43 +24,14 @@ var (
 	host_monitoring_logs_include            []string
 	host_monitoring_metrics_host_enabled    bool
 	host_monitoring_metrics_process_enabled bool
-	//go:embed observe-agent.tmpl
-	configTemplateFS embed.FS
 )
 
-const configTemplate = "observe-agent.tmpl"
-
-type FlatAgentConfig struct {
-	Token                                 string
-	ObserveURL                            string
-	CloudResourceDetectors                []string
-	SelfMonitoring_Enabled                bool
-	HostMonitoring_Enabled                bool
-	HostMonitoring_LogsEnabled            bool
-	HostMonitoring_LogsInclude            []string
-	HostMonitoring_Metrics_HostEnabled    bool
-	HostMonitoring_Metrics_ProcessEnabled bool
-}
-
 func NewConfigureCmd(v *viper.Viper) *cobra.Command {
 	return &cobra.Command{
 		Use:   "init-config",
 		Short: "Initialize agent configuration",
 		Long:  `This command takes in parameters and creates an initialized observe agent configuration file. Will overwrite existing config file and should only be used to initialize.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			configValues := FlatAgentConfig{
-				Token:                                 v.GetString("token"),
-				ObserveURL:                            v.GetString("observe_url"),
-				CloudResourceDetectors:                v.GetStringSlice("cloud_resource_detectors"),
-				SelfMonitoring_Enabled:                v.GetBool("self_monitoring::enabled"),
-				HostMonitoring_Enabled:                v.GetBool("host_monitoring::enabled"),
-				HostMonitoring_LogsEnabled:            v.GetBool("host_monitoring::logs::enabled"),
-				HostMonitoring_Metrics_HostEnabled:    v.GetBool("host_monitoring::metrics::host::enabled"),
-				HostMonitoring_Metrics_ProcessEnabled: v.GetBool("host_monitoring::metrics::process::enabled"),
-			}
-			if configValues.HostMonitoring_LogsEnabled {
-				configValues.HostMonitoring_LogsInclude = v.GetStringSlice("host_monitoring::logs::include")
-			}
 			var f *os.File
 			if v.GetBool("print") {
 				f = os.Stdout
@@ -80,10 +50,11 @@ func NewConfigureCmd(v *viper.Viper) *cobra.Command {
 				defer f.Close()
 				fmt.Printf("Writing configuration values to %s...\n\n", outputPath)
 			}
-			t := template.Must(template.New(configTemplate).ParseFS(configTemplateFS, configTemplate))
-			if err := t.ExecuteTemplate(f, configTemplate, configValues); err != nil {
+			agentConfig, err := config.AgentConfigFromViper(v)
+			if err != nil {
 				return err
 			}
+			writeConfigFile(f, agentConfig, v.GetBool("include-defaults"))
 			return nil
 		},
 	}
@@ -98,24 +69,40 @@ func init() {
 
 func RegisterConfigFlags(cmd *cobra.Command, v *viper.Viper) {
 	cmd.Flags().StringVarP(&config_path, "config_path", "", "", "Path to write config output file to")
-	cmd.PersistentFlags().Bool("print", false, "Print the configuration to stdout instead of writing to a file")
+	cmd.Flags().Bool("print", false, "Print the configuration to stdout instead of writing to a file")
+	v.BindPFlag("print", cmd.Flags().Lookup("print"))
+	cmd.Flags().Bool("include-defaults", false, "Include the names and default values for unset config options.")
+	v.BindPFlag("include-defaults", cmd.Flags().Lookup("include-defaults"))
+
 	cmd.PersistentFlags().StringVar(&token, "token", "", "Observe token")
-	cmd.PersistentFlags().StringVar(&observe_url, "observe_url", "", "Observe data collection url")
-	cmd.PersistentFlags().StringSliceVar(&cloud_resource_detectors, "cloud_resource_detectors", []string{}, "The cloud environments from which to detect resources")
-	cmd.PersistentFlags().BoolVar(&self_monitoring_enabled, "self_monitoring::enabled", true, "Enable self monitoring")
-	cmd.PersistentFlags().BoolVar(&host_monitoring_enabled, "host_monitoring::enabled", true, "Enable host monitoring")
-	cmd.PersistentFlags().BoolVar(&host_monitoring_logs_enabled, "host_monitoring::logs::enabled", true, "Enable host monitoring logs")
-	cmd.PersistentFlags().StringSliceVar(&host_monitoring_logs_include, "host_monitoring::logs::include", nil, "Set host monitoring log include paths")
-	cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_host_enabled, "host_monitoring::metrics::host::enabled", true, "Enable host monitoring host metrics")
-	cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_process_enabled, "host_monitoring::metrics::process::enabled", false, "Enable host monitoring process metrics")
-	v.BindPFlag("print", cmd.PersistentFlags().Lookup("print"))
 	v.BindPFlag("token", cmd.PersistentFlags().Lookup("token"))
+
+	cmd.PersistentFlags().StringVar(&observe_url, "observe_url", "", "Observe data collection url")
 	v.BindPFlag("observe_url", cmd.PersistentFlags().Lookup("observe_url"))
+
+	cmd.PersistentFlags().StringSliceVar(&cloud_resource_detectors, "cloud_resource_detectors", []string{}, "The cloud environments from which to detect resources")
 	v.BindPFlag("cloud_resource_detectors", cmd.PersistentFlags().Lookup("cloud_resource_detectors"))
+
+	cmd.PersistentFlags().BoolVar(&self_monitoring_enabled, "self_monitoring::enabled", true, "Enable self monitoring")
 	v.BindPFlag("self_monitoring::enabled", cmd.PersistentFlags().Lookup("self_monitoring::enabled"))
+	v.SetDefault("self_monitoring::enabled", true)
+
+	cmd.PersistentFlags().BoolVar(&host_monitoring_enabled, "host_monitoring::enabled", true, "Enable host monitoring")
 	v.BindPFlag("host_monitoring::enabled", cmd.PersistentFlags().Lookup("host_monitoring::enabled"))
+	v.SetDefault("host_monitoring::enabled", true)
+
+	cmd.PersistentFlags().BoolVar(&host_monitoring_logs_enabled, "host_monitoring::logs::enabled", true, "Enable host monitoring logs")
 	v.BindPFlag("host_monitoring::logs::enabled", cmd.PersistentFlags().Lookup("host_monitoring::logs::enabled"))
+	v.SetDefault("host_monitoring::logs::enabled", true)
+
+	cmd.PersistentFlags().StringSliceVar(&host_monitoring_logs_include, "host_monitoring::logs::include", nil, "Set host monitoring log include paths")
 	v.BindPFlag("host_monitoring::logs::include", cmd.PersistentFlags().Lookup("host_monitoring::logs::include"))
+
+	cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_host_enabled, "host_monitoring::metrics::host::enabled", true, "Enable host monitoring host metrics")
 	v.BindPFlag("host_monitoring::metrics::host::enabled", cmd.PersistentFlags().Lookup("host_monitoring::metrics::host::enabled"))
+	v.SetDefault("host_monitoring::metrics::host::enabled", true)
+
+	cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_process_enabled, "host_monitoring::metrics::process::enabled", false, "Enable host monitoring process metrics")
 	v.BindPFlag("host_monitoring::metrics::process::enabled", cmd.PersistentFlags().Lookup("host_monitoring::metrics::process::enabled"))
+	v.SetDefault("host_monitoring::metrics::process::enabled", false)
 }

internal/commands/initconfig/initconfig_test.go

@@ -5,12 +5,18 @@ import (
 	"testing"
 
 	"github.com/go-viper/mapstructure/v2"
+	"github.com/mcuadros/go-defaults"
 	"github.com/observeinc/observe-agent/internal/config"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
 
+func setConfigDefaults(agentConfig config.AgentConfig) config.AgentConfig {
+	defaults.SetDefaults(&agentConfig)
+	return agentConfig
+}
+
 func Test_InitConfigCommand(t *testing.T) {
 	t.Cleanup(func() {
 		os.Remove("./test-config.yaml")
@@ -22,7 +28,7 @@ func Test_InitConfigCommand(t *testing.T) {
 	}{
 		{
 			args: []string{"--config_path=./test-config.yaml", "--token=test-token", "--observe_url=test-url", "--host_monitoring::logs::include=/test/path,/test/path2"},
-			expectedConfig: config.AgentConfig{
+			expectedConfig: setConfigDefaults(config.AgentConfig{
 				Token:      "test-token",
 				ObserveURL: "test-url",
 				SelfMonitoring: config.SelfMonitoringConfig{
@@ -43,12 +49,12 @@ func Test_InitConfigCommand(t *testing.T) {
 						},
 					},
 				},
-			},
+			}),
 			expectErr: "",
 		},
 		{
 			args: []string{"--config_path=./test-config.yaml", "--token=test-token", "--observe_url=test-url", "--self_monitoring::enabled=false", "--host_monitoring::enabled=false", "--host_monitoring::logs::enabled=false", "--host_monitoring::metrics::host::enabled=false", "--host_monitoring::metrics::process::enabled=false"},
-			expectedConfig: config.AgentConfig{
+			expectedConfig: setConfigDefaults(config.AgentConfig{
 				Token:      "test-token",
 				ObserveURL: "test-url",
 				HostMonitoring: config.HostMonitoringConfig{
@@ -65,14 +71,15 @@ func Test_InitConfigCommand(t *testing.T) {
 						},
 					},
 				},
-			},
+			}),
 			expectErr: "",
 		},
 	}
-	v := viper.New()
-	initConfigCmd := NewConfigureCmd(v)
-	RegisterConfigFlags(initConfigCmd, v)
 	for _, tc := range testcases {
+		v := viper.NewWithOptions(viper.KeyDelimiter("::"))
+		config.SetViperDefaults(v, "::")
+		initConfigCmd := NewConfigureCmd(v)
+		RegisterConfigFlags(initConfigCmd, v)
 		initConfigCmd.SetArgs(tc.args)
 		err := initConfigCmd.Execute()
 		if err != nil {

internal/commands/initconfig/observe-agent.tmpl

@@ -0,0 +1,74 @@
+package initconfig
+
+import (
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/go-viper/mapstructure/v2"
+	"github.com/observeinc/observe-agent/internal/config"
+	"gopkg.in/yaml.v3"
+)
+
+const otelOverrideSection = `
+# otel_config_overrides:
+#   exporters:
+#     debug:
+#       verbosity: detailed
+#       sampling_initial: 5
+#       sampling_thereafter: 200
+#   service:
+#     pipelines:
+#       metrics:
+#         receivers: [hostmetrics]
+#         processors: [memory_limiter]
+#         exporters: [debug]
+`
+
+var sectionComments map[string]string = map[string]string{
+	"token":       "Observe data token",
+	"observe_url": "Target Observe collection url",
+	"debug":       "Debug mode - Sets agent log level to debug",
+	"environment": "test env comment",
+}
+
+func writeConfigFile(f *os.File, agentConfig *config.AgentConfig, includeAllOptions bool) error {
+	var yamlBytes []byte
+	var err error
+	if includeAllOptions {
+		var agentConfigMap map[string]any
+		err = mapstructure.Decode(agentConfig, &agentConfigMap)
+		if err != nil {
+			return err
+		}
+		yamlBytes, err = yaml.Marshal(&agentConfigMap)
+		if err != nil {
+			return err
+		}
+	} else {
+		yamlBytes, err = yaml.Marshal(agentConfig)
+		if err != nil {
+			return err
+		}
+	}
+	yamlStr := string(yamlBytes)
+
+	// Add empty lines between top level sections.
+	yamlStr = regexp.MustCompile(`(?m)^(\w+:.*)$`).ReplaceAllString(yamlStr, "\n$1")
+
+	// Add comments before yaml keys.
+	for commentKey, commentValue := range sectionComments {
+		yamlStr = regexp.MustCompile(`(?m)^([`+" \t"+`]*)`+commentKey+`:.*$`).ReplaceAllString(yamlStr, "$1# "+commentValue+"\n$0")
+	}
+
+	// Clean up whitespace
+	yamlStr = strings.Trim(yamlStr, " \n\t\r") + "\n"
+
+	// Add the otel config overrides comment if there is no section present.
+	if !strings.Contains(yamlStr, "otel_config_overrides:") {
+		yamlStr += "\n" + strings.Trim(otelOverrideSection, " \n\t\r") + "\n"
+	}
+
+	_, err = f.WriteString(yamlStr)
+	return err
+}

internal/commands/initconfig/writeconfigfile.go

@@ -26,7 +26,13 @@ func PrintAllConfigsIndividually(configFilePaths []string) error {
 	if err != nil {
 		return err
 	}
-	agentConfigYaml, err := yaml.Marshal(agentConfig)
+	// Use mapstructure as an intermediary so all values are printed.
+	var agentConfigMap map[string]any
+	err = mapstructure.Decode(agentConfig, &agentConfigMap)
+	if err != nil {
+		return err
+	}
+	agentConfigYaml, err := yaml.Marshal(&agentConfigMap)
 	if err != nil {
 		return err
 	}