diff --git a/.github/workflows/generate-changelog.yaml b/.github/workflows/generate-changelog.yaml index 1e057c006..1d70419e0 100644 --- a/.github/workflows/generate-changelog.yaml +++ b/.github/workflows/generate-changelog.yaml @@ -22,7 +22,7 @@ jobs: # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser-pro # 'latest', 'nightly', or a semver - version: 2.7.0 + version: 2.8.2 args: changelog env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/.github/workflows/release-build.yaml b/.github/workflows/release-build.yaml index 79f23bf37..66739feae 100644 --- a/.github/workflows/release-build.yaml +++ b/.github/workflows/release-build.yaml @@ -32,7 +32,7 @@ jobs: uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - version: 2.7.0 + version: 2.8.2 args: release --prepare --clean --snapshot --verbose env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/.github/workflows/release-nightly.yaml b/.github/workflows/release-nightly.yaml index 98bf70517..4d826a154 100644 --- a/.github/workflows/release-nightly.yaml +++ b/.github/workflows/release-nightly.yaml @@ -65,7 +65,7 @@ jobs: uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - version: 2.7.0 + version: 2.8.2 args: release --clean --skip=validate --verbose --nightly --parallelism 6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bba07fd4a..dfff07bce 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -60,7 +60,7 @@ jobs: with: distribution: goreleaser-pro # 'latest', 'nightly', or a semver - version: 2.7.0 + version: 2.8.2 args: release --clean --skip=validate --verbose env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 659a6dffd..bf02c288e 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -41,7 +41,7 @@ jobs: uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - version: 2.7.0 + version: 2.8.2 args: release --prepare --clean --snapshot --verbose --parallelism 6 env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/.github/workflows/vuln-check-full.yaml b/.github/workflows/vuln-check-full.yaml index 814ffabd0..b37f4cdd2 100644 --- a/.github/workflows/vuln-check-full.yaml +++ b/.github/workflows/vuln-check-full.yaml @@ -42,7 +42,7 @@ jobs: uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - version: 2.7.0 + version: 2.8.2 args: build --snapshot --id=default --skip=validate --single-target env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/.github/workflows/vuln-check-release.yaml b/.github/workflows/vuln-check-release.yaml index c1ea377a8..49a654733 100644 --- a/.github/workflows/vuln-check-release.yaml +++ b/.github/workflows/vuln-check-release.yaml @@ -41,7 +41,7 @@ jobs: uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - version: 2.7.0 + version: 2.8.2 args: build --snapshot --id=default --skip=validate --single-target env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml index bd4f9bb7d..327c09e84 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -16,7 +16,7 @@ before: - go generate ./... builds: - - id: default + - id: linux_build binary: observe-agent env: - CGO_ENABLED=0 @@ -44,10 +44,9 @@ builds: ldflags: -s -w -X github.com/observeinc/observe-agent/build.Version={{.Version}} archives: - - id: default + - id: linux formats: ["tar.gz"] - builds: - - default + ids: [linux_build] # this name template makes the OS and Arch compatible with the results of `uname`. name_template: >- {{ .ProjectName }}_ @@ -57,15 +56,15 @@ archives: {{- else }}{{ .Arch }}{{ end }} {{- if .Arm }}v{{ .Arm }}{{ end }} files: - - src: "packaging/linux/etc/observe-agent/*" - dst: "." + # The default agent config file and the service file live in config. - src: "packaging/linux/config/*" dst: "." + - src: "packaging/linux/connections/*" + dst: "connections" - id: windows formats: ["zip"] - builds: - - windows_build + ids: [windows_build] name_template: >- {{ .ProjectName }}_ {{- title .Os }}_ @@ -74,19 +73,14 @@ archives: {{- else }}{{ .Arch }}{{ end }} {{- if .Arm }}v{{ .Arm }}{{ end }} files: - - src: "packaging/windows/config/otel-collector.yaml" - dst: "otel-collector.yaml" - src: "packaging/windows/observe-agent.yaml" dst: "observe-agent.yaml" - - src: "packaging/windows/connections/host_monitoring/*" - dst: "connections/host_monitoring" - - src: "packaging/windows/connections/self_monitoring/*" - dst: "connections/self_monitoring" + - src: "packaging/windows/connections/*" + dst: "connections" - id: macos formats: ["zip"] - builds: - - mac_build + ids: [mac_build] name_template: >- {{ .ProjectName }}_ {{- title .Os }}_ @@ -124,7 +118,7 @@ changelog: order: 999 nfpms: - - id: default + - id: linux package_name: observe-agent @@ -168,8 +162,8 @@ nfpms: # Contents to add to the package. # GoReleaser will automatically add the binaries. contents: - - src: packaging/linux/etc/ - dst: /etc + - src: packaging/linux/connections + dst: /etc/observe-agent/connections type: tree - src: packaging/linux/config/observe-agent.yaml diff --git a/go.mod b/go.mod index 8569169e8..bcc72f727 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.7 require ( github.com/go-viper/mapstructure/v2 v2.2.1 github.com/jarcoal/httpmock v1.3.1 + github.com/mcuadros/go-defaults v1.2.0 github.com/observeinc/observe-agent/observecol v0.0.0-00010101000000-000000000000 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.62.0 diff --git a/integration/scripts/install_windows.ps1 b/integration/scripts/install_windows.ps1 index 616e12aba..ec5625b55 100644 --- a/integration/scripts/install_windows.ps1 +++ b/integration/scripts/install_windows.ps1 @@ -18,7 +18,7 @@ $temp_dir="C:\temp" #Create directories for temp & observe-agent installation ls New-Item -ItemType Directory -Force -Path $temp_dir New-Item -ItemType Directory -Force -Path $observeagent_install_dir -New-Item -ItemType Directory -Force -Path $observeagent_install_dir\config +New-Item -ItemType Directory -Force -Path $observeagent_install_dir\connections New-Item -ItemType Directory -Force -Path $program_data_filestorage # Stop the observe agent if its running so that we can copy the new .exe @@ -36,7 +36,5 @@ Expand-Archive -Force -LiteralPath $local_installer -DestinationPath "$temp_dir Write-Output "Copying files from $temp_dir\observe-agent_extract to $observeagent_install_dir" Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.exe -Destination $observeagent_install_dir Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.yaml -Destination $observeagent_install_dir -Copy-Item -Force -Path $temp_dir\observe-agent_extract\otel-collector.yaml -Destination $observeagent_install_dir\config\otel-collector.yaml -Copy-Item -Force -Path $temp_dir\observe-agent_extract\connections\ -Destination $observeagent_install_dir\connections -Recurse - - +Copy-Item -Force -Path $temp_dir\observe-agent_extract\connections\* -Destination $observeagent_install_dir\connections -Recurse +Get-ChildItem -Path $observeagent_install_dir -Recurse diff --git a/integration/scripts/start_agent_windows.ps1 b/integration/scripts/start_agent_windows.ps1 index 6c06ba2b7..bb8d45638 100644 --- a/integration/scripts/start_agent_windows.ps1 +++ b/integration/scripts/start_agent_windows.ps1 @@ -10,12 +10,19 @@ if(-not (Get-Service ObserveAgent -ErrorAction SilentlyContinue)){ DisplayName = "Observe Agent" StartupType = "Automatic" Description = "Observe Agent based on OpenTelemetry collector" - } - + } + $params | Select-Object * New-Service @params Write-Output "Starting ObserveAgent Service..." - Start-Service ObserveAgent + try { + Start-Service ObserveAgent -ErrorAction Stop + } catch { + Write-Output "Error starting ObserveAgent service!" + $_ | Select-Object * + # Print the agent config to help debug + &"${observeagent_install_dir}\observe-agent.exe" --observe-config "${observeagent_install_dir}\observe-agent.yaml" config } +} else{ Write-Output "ObserveAgent Service already exists, restarting service..." Stop-Service ObserveAgent diff --git a/internal/commands/config/config.go b/internal/commands/config/config.go index 2e9464d5d..0bf597f68 100644 --- a/internal/commands/config/config.go +++ b/internal/commands/config/config.go @@ -89,6 +89,9 @@ func printAllConfigsIndividually(configFilePaths []string) error { } func printShortOtelConfig(ctx context.Context, configFilePaths []string) error { + if len(configFilePaths) == 0 { + return nil + } settings := observecol.ConfigProviderSettings(configFilePaths) resolver, err := confmap.NewResolver(settings.ResolverSettings) if err != nil { @@ -107,6 +110,9 @@ func printShortOtelConfig(ctx context.Context, configFilePaths []string) error { } func printFullOtelConfig(configFilePaths []string) error { + if len(configFilePaths) == 0 { + return nil + } colSettings := observecol.GenerateCollectorSettingsWithConfigFiles(configFilePaths) factories, err := colSettings.Factories() if err != nil { diff --git a/internal/commands/initconfig/initconfig_test.go b/internal/commands/initconfig/initconfig_test.go index 0cf273d2c..fd60a7c6c 100644 --- a/internal/commands/initconfig/initconfig_test.go +++ b/internal/commands/initconfig/initconfig_test.go @@ -4,10 +4,11 @@ import ( "os" "testing" + "github.com/go-viper/mapstructure/v2" "github.com/observeinc/observe-agent/internal/config" "github.com/spf13/viper" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func Test_InitConfigCommand(t *testing.T) { @@ -77,17 +78,31 @@ func Test_InitConfigCommand(t *testing.T) { if err != nil { if tc.expectErr == "" { t.Errorf("Expected no error, got %v", err) + } else { + assert.ErrorContains(t, err, tc.expectErr) } } - var config config.AgentConfig - configFile, err := os.ReadFile("./test-config.yaml") - if err != nil { - t.Errorf("Expected no error, got %v", err) - } - err = yaml.Unmarshal(configFile, &config) - if err != nil { - t.Errorf("Expected no error, got %v", err) - } - assert.Equal(t, tc.expectedConfig, config) + + var configYaml, configMapstructure config.AgentConfig + + // Decode via mapstructure (which is how viper does it) + configFileContents, err := os.ReadFile("./test-config.yaml") + assert.NoError(t, err) + var yamlMap map[string]any + err = yaml.Unmarshal(configFileContents, &yamlMap) + assert.NoError(t, err) + err = mapstructure.Decode(yamlMap, &configMapstructure) + assert.NoError(t, err) + assert.Equal(t, tc.expectedConfig, configMapstructure) + + // Decode via yaml in order to do strict field checks + configFile, err := os.Open("./test-config.yaml") + assert.NoError(t, err) + decoder := yaml.NewDecoder(configFile) + // Ensure that all fields in the output yaml are present in our struct + decoder.KnownFields(true) + err = decoder.Decode(&configYaml) + assert.NoError(t, err) + assert.Equal(t, tc.expectedConfig, configYaml) } } diff --git a/internal/config/configschema.go b/internal/config/configschema.go index 54fb033a9..22642a74a 100644 --- a/internal/config/configschema.go +++ b/internal/config/configschema.go @@ -6,6 +6,7 @@ import ( "net/url" "strings" + "github.com/mcuadros/go-defaults" "github.com/spf13/viper" ) @@ -38,10 +39,21 @@ type SelfMonitoringConfig struct { Enabled bool `yaml:"enabled" mapstructure:"enabled"` } +type HealthCheckConfig struct { + Enabled bool `yaml:"enabled" mapstructure:"enabled" default:"true"` +} + +type ForwardingConfig struct { + Enabled bool `yaml:"enabled" mapstructure:"enabled" default:"true"` +} + type AgentConfig struct { Token string `yaml:"token" mapstructure:"token"` ObserveURL string `yaml:"observe_url" mapstructure:"observe_url"` CloudResourceDetectors []string `yaml:"cloud_resource_detectors,omitempty" mapstructure:"cloud_resource_detectors"` + Debug bool `yaml:"debug,omitempty" mapstructure:"debug"` + HealthCheck HealthCheckConfig `yaml:"health_check" mapstructure:"health_check"` + Forwarding ForwardingConfig `yaml:"forwarding" mapstructure:"forwarding"` SelfMonitoring SelfMonitoringConfig `yaml:"self_monitoring,omitempty" mapstructure:"self_monitoring"` HostMonitoring HostMonitoringConfig `yaml:"host_monitoring,omitempty" mapstructure:"host_monitoring"` OtelConfigOverrides map[string]any `yaml:"otel_config_overrides,omitempty" mapstructure:"otel_config_overrides"` @@ -49,6 +61,7 @@ type AgentConfig struct { func AgentConfigFromViper(v *viper.Viper) (*AgentConfig, error) { var config AgentConfig + defaults.SetDefaults(&config) err := v.Unmarshal(&config) if err != nil { return nil, err diff --git a/internal/config/configschema_test.go b/internal/config/configschema_test.go index 5cc16845e..30c49ad92 100644 --- a/internal/config/configschema_test.go +++ b/internal/config/configschema_test.go @@ -3,6 +3,7 @@ package config import ( "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) @@ -43,3 +44,26 @@ func TestAgentConfigValidate(t *testing.T) { } assert.ErrorContains(t, invalidTokenConfig.Validate(), "invalid Token") } + +func TestAgentConfigFromViper(t *testing.T) { + v := viper.NewWithOptions(viper.KeyDelimiter("::")) + v.Set("token", "some:token") + v.Set("observe_url", "https://observeinc.com") + v.Set("host_monitoring::enabled", true) + config, err := AgentConfigFromViper(v) + assert.NoError(t, err) + assert.Equal(t, "some:token", config.Token) + assert.Equal(t, "https://observeinc.com", config.ObserveURL) + assert.Equal(t, true, config.HostMonitoring.Enabled) + + // Validate that defaults are set when the value is not in the viper config + assert.Equal(t, true, config.HealthCheck.Enabled) + assert.Equal(t, true, config.Forwarding.Enabled) + + // Validate that defaults are overridden by present values + v.Set("health_check::enabled", false) + config, err = AgentConfigFromViper(v) + assert.NoError(t, err) + assert.Equal(t, false, config.HealthCheck.Enabled) + assert.Equal(t, true, config.Forwarding.Enabled) +} diff --git a/internal/connections/allconnectiontypes.go b/internal/connections/allconnectiontypes.go new file mode 100644 index 000000000..c8bded410 --- /dev/null +++ b/internal/connections/allconnectiontypes.go @@ -0,0 +1,85 @@ +package connections + +import "github.com/observeinc/observe-agent/internal/config" + +var AllConnectionTypes = []*ConnectionType{ + CommonConnectionType, + HostMonitoringConnectionType, + SelfMonitoringConnectionType, +} + +var CommonConnectionType = MakeConnectionType( + "common", + func(_ *config.AgentConfig) bool { + return true + }, + []CollectorConfigFragment{ + { + enabledCheck: func(_ *config.AgentConfig) bool { + // Always include the base connection. + return true + }, + colConfigFilePath: "base.yaml.tmpl", + }, + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.Forwarding.Enabled + }, + colConfigFilePath: "forward.yaml.tmpl", + }, + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.HealthCheck.Enabled + }, + colConfigFilePath: "health_check.yaml.tmpl", + }, + }, +) + +var HostMonitoringConnectionType = MakeConnectionType( + "host_monitoring", + func(agentConfig *config.AgentConfig) bool { + return agentConfig.HostMonitoring.Enabled + }, + []CollectorConfigFragment{ + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.HostMonitoring.Enabled + }, + colConfigFilePath: "host.yaml.tmpl", + }, + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.HostMonitoring.Metrics.Host.Enabled + }, + colConfigFilePath: "host_metrics.yaml.tmpl", + }, + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.HostMonitoring.Metrics.Process.Enabled + }, + colConfigFilePath: "process_metrics.yaml.tmpl", + }, + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.HostMonitoring.Logs.Enabled + }, + colConfigFilePath: "logs.yaml.tmpl", + }, + }, +) + +var SelfMonitoringConnectionType = MakeConnectionType( + "self_monitoring", + func(agentConfig *config.AgentConfig) bool { + return agentConfig.SelfMonitoring.Enabled + }, + []CollectorConfigFragment{ + { + enabledCheck: func(agentConfig *config.AgentConfig) bool { + return agentConfig.SelfMonitoring.Enabled + }, + colConfigFilePath: "logs_and_metrics.yaml.tmpl", + }, + }, +) diff --git a/internal/connections/confighandler.go b/internal/connections/confighandler.go index 74c9a8f69..0bc3d36f6 100644 --- a/internal/connections/confighandler.go +++ b/internal/connections/confighandler.go @@ -20,28 +20,17 @@ const ( func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, error) { configFilePaths := []string{} - // If the default otel-collector.yaml exists, add it to the list of config files - defaultOtelConfigPath := filepath.Join(GetDefaultConfigFolder(), "otel-collector.yaml") - if _, err := os.Stat(defaultOtelConfigPath); err == nil { - agentConf, err := config.AgentConfigFromViper(viper.GetViper()) - if err != nil { - return nil, err - } - otelConfigRendered, err := RenderConfigTemplate(ctx, tmpDir, defaultOtelConfigPath, agentConf) - if err != nil { - return nil, err - } - configFilePaths = append(configFilePaths, otelConfigRendered) - } // Get additional config paths based on connection configs + agentConfig, err := config.AgentConfigFromViper(viper.GetViper()) + if err != nil { + return nil, err + } for _, conn := range AllConnectionTypes { - if viper.IsSet(conn.Name) { - connectionPaths, err := conn.GetConfigFilePaths(ctx, tmpDir) - if err != nil { - return nil, err - } - configFilePaths = append(configFilePaths, connectionPaths...) + connectionPaths, err := conn.GetConfigFilePaths(ctx, tmpDir, agentConfig) + if err != nil { + return nil, err } + configFilePaths = append(configFilePaths, connectionPaths...) } // Generate override file and include path if overrides provided if viper.IsSet(OTEL_OVERRIDE_YAML_KEY) { @@ -119,19 +108,6 @@ func GetOverrideConfigFile(tmpDir string, data map[string]any) (string, error) { return f.Name(), nil } -func GetDefaultConfigFolder() string { - switch currOS := runtime.GOOS; currOS { - case "darwin": - return filepath.Join(GetDefaultAgentPath(), "config") - case "windows": - return filepath.Join(GetDefaultAgentPath(), "config") - case "linux": - return GetDefaultAgentPath() - default: - return GetDefaultAgentPath() - } -} - func GetConfigFragmentFolderPath() string { return filepath.Join(GetDefaultAgentPath(), "connections") } diff --git a/internal/connections/connections.go b/internal/connections/connections.go index ab707120f..cb2ebf1df 100644 --- a/internal/connections/connections.go +++ b/internal/connections/connections.go @@ -9,28 +9,28 @@ import ( logger "github.com/observeinc/observe-agent/internal/commands/util" "github.com/observeinc/observe-agent/internal/config" - "github.com/spf13/viper" "go.uber.org/zap" ) var TempFilesFolder = "observe-agent" +type EnabledCheckFn func(*config.AgentConfig) bool + type ConfigFieldHandler interface { GenerateCollectorConfigFragment() interface{} } type CollectorConfigFragment struct { - configYAMLPath string + enabledCheck EnabledCheckFn colConfigFilePath string } type ConnectionType struct { Name string ConfigFields []CollectorConfigFragment - Type string + EnabledCheck EnabledCheckFn configFolderPath string - getConfig func() *viper.Viper } func (c *ConnectionType) GetTemplateFilepath(tplFilename string) string { @@ -62,64 +62,37 @@ func (c *ConnectionType) RenderConfigTemplate(ctx context.Context, tmpDir string return RenderConfigTemplate(ctx, tmpDir, tplPath, confValues) } -func (c *ConnectionType) ProcessConfigFields(ctx context.Context, tmpDir string, rawConnConfig *viper.Viper, confValues any) ([]string, error) { +func (c *ConnectionType) ProcessConfigFields(ctx context.Context, tmpDir string, agentConfig *config.AgentConfig) ([]string, error) { paths := make([]string, 0) for _, field := range c.ConfigFields { - val := rawConnConfig.GetBool(field.configYAMLPath) - if val && field.colConfigFilePath != "" { - configPath, err := c.RenderConfigTemplate(ctx, tmpDir, field.colConfigFilePath, confValues) - if err != nil { - return nil, err - } - paths = append(paths, configPath) + if !field.enabledCheck(agentConfig) || field.colConfigFilePath == "" { + continue + } + configPath, err := c.RenderConfigTemplate(ctx, tmpDir, field.colConfigFilePath, agentConfig) + if err != nil { + return nil, err } + paths = append(paths, configPath) } return paths, nil } -func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string) ([]string, error) { - var rawConnConfig = c.getConfig() - var configPaths []string - if rawConnConfig == nil || !rawConnConfig.GetBool("enabled") { - return configPaths, nil +func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string, agentConfig *config.AgentConfig) ([]string, error) { + if !c.EnabledCheck(agentConfig) { + return []string{}, nil } - switch c.Type { - case SelfMonitoringConnectionTypeName: - conf := &config.SelfMonitoringConfig{} - err := rawConnConfig.Unmarshal(conf) - if err != nil { - logger.FromCtx(ctx).Error("failed to unmarshal config", zap.String("connection", c.Name)) - return nil, err - } - configPaths, err = c.ProcessConfigFields(ctx, tmpDir, rawConnConfig, conf) - if err != nil { - return nil, err - } - case HostMonitoringConnectionTypeName: - conf := &config.HostMonitoringConfig{} - err := rawConnConfig.Unmarshal(conf) - if err != nil { - logger.FromCtx(ctx).Error("failed to unmarshal config", zap.String("connection", c.Name)) - return nil, err - } - configPaths, err = c.ProcessConfigFields(ctx, tmpDir, rawConnConfig, conf) - if err != nil { - return nil, err - } - default: - logger.FromCtx(ctx).Error("unknown connection type", zap.String("type", c.Type)) - return nil, fmt.Errorf("unknown connection type %s", c.Type) + + configPaths, err := c.ProcessConfigFields(ctx, tmpDir, agentConfig) + if err != nil { + return nil, err } return configPaths, nil } type ConnectionTypeOption func(*ConnectionType) -func MakeConnectionType(Name string, ConfigFields []CollectorConfigFragment, Type string, opts ...ConnectionTypeOption) *ConnectionType { - var c = &ConnectionType{Name: Name, ConfigFields: ConfigFields, Type: Type} - c.getConfig = func() *viper.Viper { - return viper.Sub(c.Name) - } +func MakeConnectionType(name string, enabledCheck EnabledCheckFn, configFields []CollectorConfigFragment, opts ...ConnectionTypeOption) *ConnectionType { + var c = &ConnectionType{Name: name, EnabledCheck: enabledCheck, ConfigFields: configFields} c.configFolderPath = GetConfigFragmentFolderPath() // Apply provided options @@ -135,14 +108,3 @@ func WithConfigFolderPath(configFolderPath string) ConnectionTypeOption { c.configFolderPath = configFolderPath } } - -func WithGetConfig(getConfig func() *viper.Viper) ConnectionTypeOption { - return func(c *ConnectionType) { - c.getConfig = getConfig - } -} - -var AllConnectionTypes = []*ConnectionType{ - HostMonitoringConnectionType, - SelfMonitoringConnectionType, -} diff --git a/internal/connections/connections_test.go b/internal/connections/connections_test.go index e41ec889f..4a6cf787c 100644 --- a/internal/connections/connections_test.go +++ b/internal/connections/connections_test.go @@ -10,7 +10,7 @@ import ( "testing" logger "github.com/observeinc/observe-agent/internal/commands/util" - "github.com/spf13/viper" + "github.com/observeinc/observe-agent/internal/config" "github.com/stretchr/testify/suite" ) @@ -37,8 +37,14 @@ func (suite *ConnectionsTestSuite) TearDownSuite() { os.RemoveAll(suite.tempDir) } -func (suite *ConnectionsTestSuite) MakeConnectionType(configFields []CollectorConfigFragment, v *viper.Viper) *ConnectionType { - return MakeConnectionType("test", configFields, SelfMonitoringConnectionTypeName, WithConfigFolderPath(suite.configFilesPath), WithGetConfig(func() *viper.Viper { return v })) +var alwaysEnabled EnabledCheckFn = func(_ *config.AgentConfig) bool { return true } + +func (suite *ConnectionsTestSuite) MakeConnectionType(configFields []CollectorConfigFragment, enableCheck EnabledCheckFn) *ConnectionType { + return MakeConnectionType( + "test", + enableCheck, + configFields, + WithConfigFolderPath(suite.configFilesPath)) } func TestConnectionsTestSuite(t *testing.T) { @@ -47,8 +53,8 @@ func TestConnectionsTestSuite(t *testing.T) { func (suite *ConnectionsTestSuite) TestConnections_RenderConfigTemplate() { ct := suite.MakeConnectionType([]CollectorConfigFragment{ - {configYAMLPath: "field1", colConfigFilePath: "config1.tpl"}, - }, nil) + {enabledCheck: alwaysEnabled, colConfigFilePath: "config1.tpl"}, + }, alwaysEnabled) // Test the RenderConfigTemplate function confValues := struct { @@ -84,20 +90,18 @@ func (suite *ConnectionsTestSuite) TestConnections_RenderConfigTemplate() { } func (suite *ConnectionsTestSuite) TestConnectionType_ProcessConfigFields() { - // Mock viper configuration - v := viper.New() - v.Set("field1", true) - v.Set("field2", false) - v.Set("field3", true) + var agentConfig config.AgentConfig + agentConfig.HostMonitoring.Enabled = true + agentConfig.SelfMonitoring.Enabled = false + agentConfig.Forwarding.Enabled = true ct := suite.MakeConnectionType([]CollectorConfigFragment{ - {configYAMLPath: "field1", colConfigFilePath: "testConfig1.tpl"}, - {configYAMLPath: "field2", colConfigFilePath: "testConfig2.tpl"}, - {configYAMLPath: "field3", colConfigFilePath: ""}, - }, v) + {enabledCheck: func(ac *config.AgentConfig) bool { return ac.HostMonitoring.Enabled }, colConfigFilePath: "testConfig1.tpl"}, + {enabledCheck: func(ac *config.AgentConfig) bool { return ac.SelfMonitoring.Enabled }, colConfigFilePath: "testConfig2.tpl"}, + {enabledCheck: func(ac *config.AgentConfig) bool { return ac.Forwarding.Enabled }, colConfigFilePath: ""}, + }, alwaysEnabled) - confValues := struct{}{} - paths, err := ct.ProcessConfigFields(suite.ctx, suite.tempDir, v, confValues) + paths, err := ct.ProcessConfigFields(suite.ctx, suite.tempDir, &agentConfig) suite.NoError(err) suite.Len(paths, 1) @@ -107,20 +111,19 @@ func (suite *ConnectionsTestSuite) TestConnectionType_ProcessConfigFields() { } func (suite *ConnectionsTestSuite) TestConnectionType_GetConfigFilePaths() { - // Mock viper configuration - v := viper.New() - v.Set("enabled", true) - v.Set("field1", true) - v.Set("field2", false) - v.Set("field3", false) + var agentConfig config.AgentConfig + agentConfig.Debug = true + agentConfig.HostMonitoring.Enabled = true + agentConfig.SelfMonitoring.Enabled = false + agentConfig.Forwarding.Enabled = true ct := suite.MakeConnectionType([]CollectorConfigFragment{ - {configYAMLPath: "field1", colConfigFilePath: "testConfig1.tpl"}, - {configYAMLPath: "field2", colConfigFilePath: "testConfig2.tpl"}, - {configYAMLPath: "field3", colConfigFilePath: ""}, - }, v) + {enabledCheck: func(ac *config.AgentConfig) bool { return ac.HostMonitoring.Enabled }, colConfigFilePath: "testConfig1.tpl"}, + {enabledCheck: func(ac *config.AgentConfig) bool { return ac.SelfMonitoring.Enabled }, colConfigFilePath: "testConfig2.tpl"}, + {enabledCheck: func(ac *config.AgentConfig) bool { return ac.Forwarding.Enabled }, colConfigFilePath: ""}, + }, func(ac *config.AgentConfig) bool { return ac.Debug }) - paths, err := ct.GetConfigFilePaths(suite.ctx, suite.tempDir) + paths, err := ct.GetConfigFilePaths(suite.ctx, suite.tempDir, &agentConfig) suite.NoError(err) suite.Len(paths, 1) tmpFile := paths[0] @@ -128,8 +131,8 @@ func (suite *ConnectionsTestSuite) TestConnectionType_GetConfigFilePaths() { suite.Equal(ct.ConfigFields[0].colConfigFilePath, tmpConfName) // Does nothing if not enabled - v.Set("enabled", false) - paths, err = ct.GetConfigFilePaths(suite.ctx, suite.tempDir) + agentConfig.Debug = false + paths, err = ct.GetConfigFilePaths(suite.ctx, suite.tempDir, &agentConfig) suite.NoError(err) suite.Len(paths, 0) } diff --git a/internal/connections/hostmonitoring.go b/internal/connections/hostmonitoring.go deleted file mode 100644 index 1ca3c98cb..000000000 --- a/internal/connections/hostmonitoring.go +++ /dev/null @@ -1,30 +0,0 @@ -package connections - -var HostMonitoringConnectionTypeName = "host_monitoring" - -var HostMonitoringConnectionType = MakeConnectionType( - "host_monitoring", - []CollectorConfigFragment{ - { - configYAMLPath: "enabled", - colConfigFilePath: "host.yaml", - }, - { - configYAMLPath: "metrics::enabled", - colConfigFilePath: "host_metrics.yaml", - }, - { - configYAMLPath: "metrics::host::enabled", - colConfigFilePath: "host_metrics.yaml", - }, - { - configYAMLPath: "metrics::process::enabled", - colConfigFilePath: "process_metrics.yaml", - }, - { - configYAMLPath: "logs::enabled", - colConfigFilePath: "logs.yaml", - }, - }, - HostMonitoringConnectionTypeName, -) diff --git a/internal/connections/selfmonitoring.go b/internal/connections/selfmonitoring.go deleted file mode 100644 index 65594d94b..000000000 --- a/internal/connections/selfmonitoring.go +++ /dev/null @@ -1,14 +0,0 @@ -package connections - -var SelfMonitoringConnectionTypeName = "self_monitoring" - -var SelfMonitoringConnectionType = MakeConnectionType( - "self_monitoring", - []CollectorConfigFragment{ - { - configYAMLPath: "enabled", - colConfigFilePath: "logs_and_metrics.yaml", - }, - }, - SelfMonitoringConnectionTypeName, -) diff --git a/internal/connections/templatefuncs.go b/internal/connections/templatefuncs.go index c2945131e..c30e2f861 100644 --- a/internal/connections/templatefuncs.go +++ b/internal/connections/templatefuncs.go @@ -12,7 +12,7 @@ func GetTemplateFuncMap() template.FuncMap { return template.FuncMap{ "inlineArrayInt": TplInlineArray[int], "inlineArrayStr": TplInlineArray[string], - "valToYaml": TmplValueToYaml, + "valToYaml": TplValueToYaml, "objToYaml": TplToYaml, "add": func(values ...int) int { sum := 0 @@ -27,12 +27,12 @@ func GetTemplateFuncMap() template.FuncMap { func TplInlineArray[T any](arr []T) string { strs := make([]string, len(arr)) for i := range arr { - strs[i] = TmplValueToYaml(arr[i]) + strs[i] = TplValueToYaml(arr[i]) } return "[" + strings.Join(strs, ", ") + "]" } -func TmplValueToYaml(value any) string { +func TplValueToYaml(value any) string { b, err := yaml.Marshal(value) if err != nil { panic(err) diff --git a/internal/connections/templatefuncs_test.go b/internal/connections/templatefuncs_test.go index 0ac2acf83..195859d18 100644 --- a/internal/connections/templatefuncs_test.go +++ b/internal/connections/templatefuncs_test.go @@ -6,12 +6,12 @@ import ( "github.com/stretchr/testify/assert" ) -func TestTmplValueToYaml(t *testing.T) { - assert.Equal(t, "\"y\"", TmplValueToYaml("y")) - assert.Equal(t, "a", TmplValueToYaml("a")) - assert.Equal(t, "5", TmplValueToYaml(5)) - assert.Equal(t, "2.22", TmplValueToYaml(2.22)) - assert.Equal(t, "true", TmplValueToYaml(true)) +func TestTplValueToYaml(t *testing.T) { + assert.Equal(t, "\"y\"", TplValueToYaml("y")) + assert.Equal(t, "a", TplValueToYaml("a")) + assert.Equal(t, "5", TplValueToYaml(5)) + assert.Equal(t, "2.22", TplValueToYaml(2.22)) + assert.Equal(t, "true", TplValueToYaml(true)) } func TestTplInlineArray(t *testing.T) { diff --git a/packaging/docker/observe-agent/otel-collector.yaml b/packaging/docker/observe-agent/connections/common/base.yaml.tmpl similarity index 73% rename from packaging/docker/observe-agent/otel-collector.yaml rename to packaging/docker/observe-agent/connections/common/base.yaml.tmpl index 15e2466da..9bd9bc23c 100644 --- a/packaging/docker/observe-agent/otel-collector.yaml +++ b/packaging/docker/observe-agent/connections/common/base.yaml.tmpl @@ -1,21 +1,10 @@ extensions: - health_check: - endpoint: 'localhost:13133' - path: '/status' file_storage: directory: ${env:FILESTORAGE_PATH} connectors: count: -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - processors: # Snowflake limit for identifiers: Regardless of whether an identifier is unquoted or double-quoted, the maximum number of characters allowed is 255 (including blank spaces). # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#identifier-requirements @@ -106,37 +95,10 @@ exporters: enabled: true # Convert resource attributes to metric labels send_metadata: true - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - debug: service: pipelines: - metrics/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud, deltatocumulative, batch] - exporters: [prometheusremotewrite/observe] - - logs/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud] - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud] - exporters: [otlphttp/observetracing] - metrics/count-nooop: receivers: [count] processors: [filter/count] diff --git a/packaging/docker/observe-agent/connections/common/forward.yaml.tmpl b/packaging/docker/observe-agent/connections/common/forward.yaml.tmpl new file mode 100644 index 000000000..99b4e1436 --- /dev/null +++ b/packaging/docker/observe-agent/connections/common/forward.yaml.tmpl @@ -0,0 +1,37 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + http: + endpoint: localhost:4318 + +exporters: + otlphttp/observetracing: + endpoint: ${env:OBSERVE_OTEL_ENDPOINT} + headers: + authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} + x-observe-target-package: "Tracing" + sending_queue: + num_consumers: 4 + queue_size: 100 + retry_on_failure: + enabled: true + compression: zstd + +service: + pipelines: + metrics/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud, deltatocumulative, batch] + exporters: [prometheusremotewrite/observe] + + logs/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud] + exporters: [otlphttp/observe, count] + + traces/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud] + exporters: [otlphttp/observetracing] diff --git a/packaging/docker/observe-agent/connections/common/health_check.yaml.tmpl b/packaging/docker/observe-agent/connections/common/health_check.yaml.tmpl new file mode 100644 index 000000000..98e7d5126 --- /dev/null +++ b/packaging/docker/observe-agent/connections/common/health_check.yaml.tmpl @@ -0,0 +1,4 @@ +extensions: + health_check: + endpoint: 'localhost:13133' + path: '/status' diff --git a/packaging/macos/connections/host_monitoring/host.yaml b/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl similarity index 75% rename from packaging/macos/connections/host_monitoring/host.yaml rename to packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl index 66494efe5..e5e382eb0 100644 --- a/packaging/macos/connections/host_monitoring/host.yaml +++ b/packaging/docker/observe-agent/connections/host_monitoring/host.yaml.tmpl @@ -1,6 +1,7 @@ receivers: + # TODO deprecate this filestats/agent: - include: '/usr/local/observe-agent/config/otel-collector.yaml' + include: '/etc/observe-agent/connections/common/base.yaml.tmpl' collection_interval: 5m initial_delay: 60s diff --git a/packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml b/packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml rename to packaging/docker/observe-agent/connections/host_monitoring/host_metrics.yaml.tmpl diff --git a/packaging/docker/observe-agent/connections/host_monitoring/logs.yaml b/packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl similarity index 86% rename from packaging/docker/observe-agent/connections/host_monitoring/logs.yaml rename to packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl index 88b11bc3d..2dfa1187d 100644 --- a/packaging/docker/observe-agent/connections/host_monitoring/logs.yaml +++ b/packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl @@ -1,17 +1,17 @@ receivers: filelog/host_monitoring: include: - {{- if .Logs.Include }} - {{- range .Logs.Include }} + {{- if .HostMonitoring.Logs.Include }} + {{- range .HostMonitoring.Logs.Include }} - {{ . }} {{- end }} {{- else }} - /hostfs/var/log/**/*.log - /hostfs/var/log/syslog {{- end }} - {{- if .Logs.Exclude }} + {{- if .HostMonitoring.Logs.Exclude }} exclude: - {{- range .Logs.Exclude }} + {{- range .HostMonitoring.Logs.Exclude }} - {{ . }} {{- end }} {{- end }} diff --git a/packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml b/packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml.tmpl similarity index 100% rename from packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml rename to packaging/docker/observe-agent/connections/host_monitoring/process_metrics.yaml.tmpl diff --git a/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml b/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 93% rename from packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml rename to packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 95f940931..51f2df3cc 100644 --- a/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml +++ b/packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,7 @@ receivers: filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - include: [/etc/observe-agent/otel-collector.yaml] + # TODO deprecate this + include: [/etc/observe-agent/connections/common/base.yaml.tmpl] include_file_path: true start_at: beginning poll_interval: 5m diff --git a/packaging/linux/etc/observe-agent/otel-collector.yaml b/packaging/linux/connections/common/base.yaml.tmpl similarity index 73% rename from packaging/linux/etc/observe-agent/otel-collector.yaml rename to packaging/linux/connections/common/base.yaml.tmpl index 15e2466da..9bd9bc23c 100644 --- a/packaging/linux/etc/observe-agent/otel-collector.yaml +++ b/packaging/linux/connections/common/base.yaml.tmpl @@ -1,21 +1,10 @@ extensions: - health_check: - endpoint: 'localhost:13133' - path: '/status' file_storage: directory: ${env:FILESTORAGE_PATH} connectors: count: -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - processors: # Snowflake limit for identifiers: Regardless of whether an identifier is unquoted or double-quoted, the maximum number of characters allowed is 255 (including blank spaces). # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#identifier-requirements @@ -106,37 +95,10 @@ exporters: enabled: true # Convert resource attributes to metric labels send_metadata: true - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - debug: service: pipelines: - metrics/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud, deltatocumulative, batch] - exporters: [prometheusremotewrite/observe] - - logs/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud] - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud] - exporters: [otlphttp/observetracing] - metrics/count-nooop: receivers: [count] processors: [filter/count] diff --git a/packaging/linux/connections/common/forward.yaml.tmpl b/packaging/linux/connections/common/forward.yaml.tmpl new file mode 100644 index 000000000..99b4e1436 --- /dev/null +++ b/packaging/linux/connections/common/forward.yaml.tmpl @@ -0,0 +1,37 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + http: + endpoint: localhost:4318 + +exporters: + otlphttp/observetracing: + endpoint: ${env:OBSERVE_OTEL_ENDPOINT} + headers: + authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} + x-observe-target-package: "Tracing" + sending_queue: + num_consumers: 4 + queue_size: 100 + retry_on_failure: + enabled: true + compression: zstd + +service: + pipelines: + metrics/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud, deltatocumulative, batch] + exporters: [prometheusremotewrite/observe] + + logs/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud] + exporters: [otlphttp/observe, count] + + traces/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud] + exporters: [otlphttp/observetracing] diff --git a/packaging/linux/connections/common/health_check.yaml.tmpl b/packaging/linux/connections/common/health_check.yaml.tmpl new file mode 100644 index 000000000..98e7d5126 --- /dev/null +++ b/packaging/linux/connections/common/health_check.yaml.tmpl @@ -0,0 +1,4 @@ +extensions: + health_check: + endpoint: 'localhost:13133' + path: '/status' diff --git a/packaging/windows/connections/host_monitoring/host.yaml b/packaging/linux/connections/host_monitoring/host.yaml.tmpl similarity index 75% rename from packaging/windows/connections/host_monitoring/host.yaml rename to packaging/linux/connections/host_monitoring/host.yaml.tmpl index bead927e8..e5e382eb0 100644 --- a/packaging/windows/connections/host_monitoring/host.yaml +++ b/packaging/linux/connections/host_monitoring/host.yaml.tmpl @@ -1,6 +1,7 @@ receivers: + # TODO deprecate this filestats/agent: - include: 'C:\Program Files\Observe\observe-agent\config\otel-collector.yaml' + include: '/etc/observe-agent/connections/common/base.yaml.tmpl' collection_interval: 5m initial_delay: 60s diff --git a/packaging/linux/etc/observe-agent/connections/host_monitoring/host_metrics.yaml b/packaging/linux/connections/host_monitoring/host_metrics.yaml.tmpl similarity index 100% rename from packaging/linux/etc/observe-agent/connections/host_monitoring/host_metrics.yaml rename to packaging/linux/connections/host_monitoring/host_metrics.yaml.tmpl diff --git a/packaging/linux/etc/observe-agent/connections/host_monitoring/logs.yaml b/packaging/linux/connections/host_monitoring/logs.yaml.tmpl similarity index 86% rename from packaging/linux/etc/observe-agent/connections/host_monitoring/logs.yaml rename to packaging/linux/connections/host_monitoring/logs.yaml.tmpl index 0f203036b..e2ecf6f52 100644 --- a/packaging/linux/etc/observe-agent/connections/host_monitoring/logs.yaml +++ b/packaging/linux/connections/host_monitoring/logs.yaml.tmpl @@ -1,17 +1,17 @@ receivers: filelog/host_monitoring: include: - {{- if .Logs.Include }} - {{- range .Logs.Include }} + {{- if .HostMonitoring.Logs.Include }} + {{- range .HostMonitoring.Logs.Include }} - {{ . }} {{- end }} {{- else }} - /var/log/**/*.log - /var/log/syslog {{- end }} - {{- if .Logs.Exclude }} + {{- if .HostMonitoring.Logs.Exclude }} exclude: - {{- range .Logs.Exclude }} + {{- range .HostMonitoring.Logs.Exclude }} - {{ . }} {{- end }} {{- end }} diff --git a/packaging/linux/etc/observe-agent/connections/host_monitoring/process_metrics.yaml b/packaging/linux/connections/host_monitoring/process_metrics.yaml.tmpl similarity index 100% rename from packaging/linux/etc/observe-agent/connections/host_monitoring/process_metrics.yaml rename to packaging/linux/connections/host_monitoring/process_metrics.yaml.tmpl diff --git a/packaging/linux/etc/observe-agent/connections/self_monitoring/logs_and_metrics.yaml b/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 93% rename from packaging/linux/etc/observe-agent/connections/self_monitoring/logs_and_metrics.yaml rename to packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 95f940931..51f2df3cc 100644 --- a/packaging/linux/etc/observe-agent/connections/self_monitoring/logs_and_metrics.yaml +++ b/packaging/linux/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,7 @@ receivers: filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - include: [/etc/observe-agent/otel-collector.yaml] + # TODO deprecate this + include: [/etc/observe-agent/connections/common/base.yaml.tmpl] include_file_path: true start_at: beginning poll_interval: 5m diff --git a/packaging/macos/config/otel-collector.yaml b/packaging/macos/connections/common/base.yaml.tmpl similarity index 73% rename from packaging/macos/config/otel-collector.yaml rename to packaging/macos/connections/common/base.yaml.tmpl index 15e2466da..9bd9bc23c 100644 --- a/packaging/macos/config/otel-collector.yaml +++ b/packaging/macos/connections/common/base.yaml.tmpl @@ -1,21 +1,10 @@ extensions: - health_check: - endpoint: 'localhost:13133' - path: '/status' file_storage: directory: ${env:FILESTORAGE_PATH} connectors: count: -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - processors: # Snowflake limit for identifiers: Regardless of whether an identifier is unquoted or double-quoted, the maximum number of characters allowed is 255 (including blank spaces). # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#identifier-requirements @@ -106,37 +95,10 @@ exporters: enabled: true # Convert resource attributes to metric labels send_metadata: true - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - debug: service: pipelines: - metrics/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud, deltatocumulative, batch] - exporters: [prometheusremotewrite/observe] - - logs/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud] - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: [resourcedetection, resourcedetection/cloud] - exporters: [otlphttp/observetracing] - metrics/count-nooop: receivers: [count] processors: [filter/count] diff --git a/packaging/macos/connections/common/forward.yaml.tmpl b/packaging/macos/connections/common/forward.yaml.tmpl new file mode 100644 index 000000000..99b4e1436 --- /dev/null +++ b/packaging/macos/connections/common/forward.yaml.tmpl @@ -0,0 +1,37 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + http: + endpoint: localhost:4318 + +exporters: + otlphttp/observetracing: + endpoint: ${env:OBSERVE_OTEL_ENDPOINT} + headers: + authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} + x-observe-target-package: "Tracing" + sending_queue: + num_consumers: 4 + queue_size: 100 + retry_on_failure: + enabled: true + compression: zstd + +service: + pipelines: + metrics/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud, deltatocumulative, batch] + exporters: [prometheusremotewrite/observe] + + logs/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud] + exporters: [otlphttp/observe, count] + + traces/forward: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud] + exporters: [otlphttp/observetracing] diff --git a/packaging/macos/connections/common/health_check.yaml.tmpl b/packaging/macos/connections/common/health_check.yaml.tmpl new file mode 100644 index 000000000..98e7d5126 --- /dev/null +++ b/packaging/macos/connections/common/health_check.yaml.tmpl @@ -0,0 +1,4 @@ +extensions: + health_check: + endpoint: 'localhost:13133' + path: '/status' diff --git a/packaging/docker/observe-agent/connections/host_monitoring/host.yaml b/packaging/macos/connections/host_monitoring/host.yaml.tmpl similarity index 74% rename from packaging/docker/observe-agent/connections/host_monitoring/host.yaml rename to packaging/macos/connections/host_monitoring/host.yaml.tmpl index dce770063..5c4f1b443 100644 --- a/packaging/docker/observe-agent/connections/host_monitoring/host.yaml +++ b/packaging/macos/connections/host_monitoring/host.yaml.tmpl @@ -1,6 +1,7 @@ receivers: + # TODO deprecate this filestats/agent: - include: '/etc/observe-agent/otel-collector.yaml' + include: '/usr/local/observe-agent/connections/common/base.yaml.tmpl' collection_interval: 5m initial_delay: 60s diff --git a/packaging/macos/connections/host_monitoring/host_metrics.yaml b/packaging/macos/connections/host_monitoring/host_metrics.yaml.tmpl similarity index 100% rename from packaging/macos/connections/host_monitoring/host_metrics.yaml rename to packaging/macos/connections/host_monitoring/host_metrics.yaml.tmpl diff --git a/packaging/macos/connections/host_monitoring/logs.yaml b/packaging/macos/connections/host_monitoring/logs.yaml.tmpl similarity index 77% rename from packaging/macos/connections/host_monitoring/logs.yaml rename to packaging/macos/connections/host_monitoring/logs.yaml.tmpl index 917c47a11..90f0559f0 100644 --- a/packaging/macos/connections/host_monitoring/logs.yaml +++ b/packaging/macos/connections/host_monitoring/logs.yaml.tmpl @@ -1,16 +1,16 @@ receivers: filelog/host_monitoring: include: - {{- if .Logs.Include }} - {{- range .Logs.Include }} + {{- if .HostMonitoring.Logs.Include }} + {{- range .HostMonitoring.Logs.Include }} - {{ . }} {{- end }} {{- else }} - /var/log/**/*.log {{- end }} - {{- if .Logs.Exclude }} + {{- if .HostMonitoring.Logs.Exclude }} exclude: - {{- range .Logs.Exclude }} + {{- range .HostMonitoring.Logs.Exclude }} - {{ . }} {{- end }} {{- end }} diff --git a/packaging/macos/connections/host_monitoring/process_metrics.yaml b/packaging/macos/connections/host_monitoring/process_metrics.yaml.tmpl similarity index 100% rename from packaging/macos/connections/host_monitoring/process_metrics.yaml rename to packaging/macos/connections/host_monitoring/process_metrics.yaml.tmpl diff --git a/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml b/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 91% rename from packaging/macos/connections/self_monitoring/logs_and_metrics.yaml rename to packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl index 4411c6761..fba2fe27d 100644 --- a/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml +++ b/packaging/macos/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,7 @@ receivers: filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - include: [/usr/local/observe-agent/config/otel-collector.yaml] + # TODO deprecate this + include: [/usr/local/observe-agent/connections/common/base.yaml.tmpl] include_file_path: true start_at: beginning poll_interval: 5m diff --git a/packaging/windows/config/otel-collector.yaml b/packaging/windows/connections/common/base.yaml.tmpl similarity index 67% rename from packaging/windows/config/otel-collector.yaml rename to packaging/windows/connections/common/base.yaml.tmpl index ccefd9ed0..4614beec5 100644 --- a/packaging/windows/config/otel-collector.yaml +++ b/packaging/windows/connections/common/base.yaml.tmpl @@ -1,21 +1,10 @@ extensions: - health_check: - endpoint: 'localhost:13133' - path: '/status' file_storage: directory: ${env:FILESTORAGE_PATH} connectors: count: -receivers: - otlp: - protocols: - grpc: - endpoint: localhost:4317 - http: - endpoint: localhost:4318 - processors: # Snowflake limit for identifiers: Regardless of whether an identifier is unquoted or double-quoted, the maximum number of characters allowed is 255 (including blank spaces). # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#identifier-requirements @@ -86,37 +75,10 @@ exporters: enabled: true # Convert resource attributes to metric labels send_metadata: true - otlphttp/observetracing: - endpoint: ${env:OBSERVE_OTEL_ENDPOINT} - headers: - authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} - x-observe-target-package: "Tracing" - sending_queue: - num_consumers: 4 - queue_size: 100 - retry_on_failure: - enabled: true - compression: zstd - debug: service: pipelines: - metrics/forward: - receivers: [otlp] - processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, deltatocumulative, batch] - exporters: [prometheusremotewrite/observe] - - logs/forward: - receivers: [otlp] - processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch] - exporters: [otlphttp/observe, count] - - traces/forward: - receivers: [otlp] - processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch] - exporters: [otlphttp/observetracing] - metrics/count-nooop: receivers: [count] processors: [filter/count] diff --git a/packaging/windows/connections/common/forward.yaml.tmpl b/packaging/windows/connections/common/forward.yaml.tmpl new file mode 100644 index 000000000..7797068d8 --- /dev/null +++ b/packaging/windows/connections/common/forward.yaml.tmpl @@ -0,0 +1,37 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + http: + endpoint: localhost:4318 + +exporters: + otlphttp/observetracing: + endpoint: ${env:OBSERVE_OTEL_ENDPOINT} + headers: + authorization: ${env:OBSERVE_AUTHORIZATION_HEADER} + x-observe-target-package: "Tracing" + sending_queue: + num_consumers: 4 + queue_size: 100 + retry_on_failure: + enabled: true + compression: zstd + +service: + pipelines: + metrics/forward: + receivers: [otlp] + processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, deltatocumulative, batch] + exporters: [prometheusremotewrite/observe] + + logs/forward: + receivers: [otlp] + processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch] + exporters: [otlphttp/observe, count] + + traces/forward: + receivers: [otlp] + processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch] + exporters: [otlphttp/observetracing] diff --git a/packaging/windows/connections/common/health_check.yaml.tmpl b/packaging/windows/connections/common/health_check.yaml.tmpl new file mode 100644 index 000000000..98e7d5126 --- /dev/null +++ b/packaging/windows/connections/common/health_check.yaml.tmpl @@ -0,0 +1,4 @@ +extensions: + health_check: + endpoint: 'localhost:13133' + path: '/status' diff --git a/packaging/linux/etc/observe-agent/connections/host_monitoring/host.yaml b/packaging/windows/connections/host_monitoring/host.yaml.tmpl similarity index 71% rename from packaging/linux/etc/observe-agent/connections/host_monitoring/host.yaml rename to packaging/windows/connections/host_monitoring/host.yaml.tmpl index dce770063..827a12a12 100644 --- a/packaging/linux/etc/observe-agent/connections/host_monitoring/host.yaml +++ b/packaging/windows/connections/host_monitoring/host.yaml.tmpl @@ -1,6 +1,7 @@ receivers: + # TODO deprecate this filestats/agent: - include: '/etc/observe-agent/otel-collector.yaml' + include: 'C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl' collection_interval: 5m initial_delay: 60s diff --git a/packaging/windows/connections/host_monitoring/host_metrics.yaml b/packaging/windows/connections/host_monitoring/host_metrics.yaml.tmpl similarity index 100% rename from packaging/windows/connections/host_monitoring/host_metrics.yaml rename to packaging/windows/connections/host_monitoring/host_metrics.yaml.tmpl diff --git a/packaging/windows/connections/host_monitoring/logs.yaml b/packaging/windows/connections/host_monitoring/logs.yaml.tmpl similarity index 100% rename from packaging/windows/connections/host_monitoring/logs.yaml rename to packaging/windows/connections/host_monitoring/logs.yaml.tmpl diff --git a/packaging/windows/connections/host_monitoring/process_metrics.yaml b/packaging/windows/connections/host_monitoring/process_metrics.yaml.tmpl similarity index 100% rename from packaging/windows/connections/host_monitoring/process_metrics.yaml rename to packaging/windows/connections/host_monitoring/process_metrics.yaml.tmpl diff --git a/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml b/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl similarity index 89% rename from packaging/windows/connections/self_monitoring/logs_and_metrics.yaml rename to packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl index e7940130d..09a2920e2 100644 --- a/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml +++ b/packaging/windows/connections/self_monitoring/logs_and_metrics.yaml.tmpl @@ -1,6 +1,7 @@ receivers: filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields - include: ['C:\Program Files\Observe\observe-agent\config\otel-collector.yaml'] + # TODO deprecate this + include: ['C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl'] include_file_path: true start_at: beginning poll_interval: 5m diff --git a/scripts/install.ps1 b/scripts/install.ps1 index e8b682a96..8554f1e67 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -21,7 +21,6 @@ $temp_dir="C:\temp" New-Item -ItemType Directory -Force -Path $temp_dir New-Item -ItemType Directory -Force -Path $observeagent_install_dir -New-Item -ItemType Directory -Force -Path $observeagent_install_dir\config New-Item -ItemType Directory -Force -Path $observeagent_install_dir\connections New-Item -ItemType Directory -Force -Path $program_data_filestorage @@ -34,7 +33,6 @@ if((Get-Service ObserveAgent -ErrorAction SilentlyContinue)){ Expand-Archive -Force -LiteralPath $local_installer -DestinationPath "$temp_dir\observe-agent_Windows_x86_64" Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\observe-agent.exe -Destination $observeagent_install_dir -Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\otel-collector.yaml -Destination $observeagent_install_dir\config\otel-collector.yaml Copy-Item -Force -Path $temp_dir\observe-agent_Windows_x86_64\connections\* -Destination $observeagent_install_dir\connections -Recurse # If there's already an observe-agent.yaml we don't copy the template from the downloaded installation package and leave existing observe-agent.yaml alone diff --git a/scripts/install_linux.sh b/scripts/install_linux.sh index da0fd8980..5fdbfd2ce 100644 --- a/scripts/install_linux.sh +++ b/scripts/install_linux.sh @@ -93,8 +93,7 @@ sudo chmod +rw /var/lib/observe-agent/filestorage sudo cp -f $tmp_dir/observe-agent $agent_binary_path # Copy all config files to the proper dir. -sudo cp -f $tmp_dir/otel-collector.yaml $observeagent_config_dir/otel-collector.yaml -sudo rm -rf $observeagent_config_dir/connections +sudo rm -rf $observeagent_config_dir/otel-collector.yaml $observeagent_config_dir/connections sudo cp -fR $tmp_dir/connections $observeagent_config_dir/connections sudo chown -R root:root $observeagent_config_dir diff --git a/scripts/install_mac.sh b/scripts/install_mac.sh index dbcead55f..5357c0cec 100644 --- a/scripts/install_mac.sh +++ b/scripts/install_mac.sh @@ -101,7 +101,6 @@ sudo chmod +rw /var/lib/observe-agent/filestorage # Copy all files to the install dir. sudo rm -rf $observeagent_install_dir/config $observeagent_install_dir/connections $observeagent_install_dir/observe-agent sudo cp $tmp_dir/observe-agent $observeagent_install_dir/observe-agent -sudo cp -R $tmp_dir/config $observeagent_install_dir/config sudo cp -R $tmp_dir/connections $observeagent_install_dir/connections sudo chown -R root:wheel $observeagent_install_dir diff --git a/vendor/github.com/mcuadros/go-defaults/LICENSE b/vendor/github.com/mcuadros/go-defaults/LICENSE new file mode 100644 index 000000000..93cb2af62 --- /dev/null +++ b/vendor/github.com/mcuadros/go-defaults/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2014 Máximo Cuadros + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mcuadros/go-defaults/README.md b/vendor/github.com/mcuadros/go-defaults/README.md new file mode 100644 index 000000000..766015da6 --- /dev/null +++ b/vendor/github.com/mcuadros/go-defaults/README.md @@ -0,0 +1,72 @@ +go-defaults [![Build Status](https://img.shields.io/github/workflow/status/mcuadros/go-defaults/Test.svg)](https://github.com/mcuadros/go-defaults/actions) [![GoDoc](http://godoc.org/github.com/mcuadros/go-defaults?status.png)](https://pkg.go.dev/github.com/mcuadros/go-defaults) [![GitHub release](https://img.shields.io/github/release/mcuadros/go-defaults.svg)](https://github.com/mcuadros/go-defaults/releases) +============================== + +Enabling stuctures with defaults values using [struct tags](http://golang.org/pkg/reflect/#StructTag). + +Installation +------------ + +The recommended way to install go-defaults + +``` +go get gopkg.in/mcuadros/go-defaults.v1 +``` + +Examples +-------- + +A basic example: + +```go +import ( + "fmt" + "github.com/mcuadros/go-defaults" + "time" +) + +type ExampleBasic struct { + Foo bool `default:"true"` //<-- StructTag with a default key + Bar string `default:"33"` + Qux int8 + Dur time.Duration `default:"1m"` +} + +func NewExampleBasic() *ExampleBasic { + example := new(ExampleBasic) + defaults.SetDefaults(example) //<-- This set the defaults values + + return example +} + +... + +test := NewExampleBasic() +fmt.Println(test.Foo) //Prints: true +fmt.Println(test.Bar) //Prints: 33 +fmt.Println(test.Qux) //Prints: +fmt.Println(test.Dur) //Prints: 1m0s +``` + +Caveats +------- + +At the moment, the way the default filler checks whether it should fill a struct field or not is by comparing the current field value with the corresponding zero value of that type. This has a subtle implication: the zero value set explicitly by you will get overriden by default value during `SetDefaults()` call. So if you need to set the field to container zero value, you need to set it explicitly AFTER setting the defaults. + +Take the basic example in the above section and change it slightly: +```go + +example := ExampleBasic{ + Bar: 0, +} +defaults.SetDefaults(example) +fmt.Println(example.Bar) //Prints: 33 instead of 0 (which is zero value for int) + +example.Bar = 0 // set needed zero value AFTER applying defaults +fmt.Println(example.Bar) //Prints: 0 + +``` + +License +------- + +MIT, see [LICENSE](LICENSE) diff --git a/vendor/github.com/mcuadros/go-defaults/defaults.go b/vendor/github.com/mcuadros/go-defaults/defaults.go new file mode 100644 index 000000000..3666aba9f --- /dev/null +++ b/vendor/github.com/mcuadros/go-defaults/defaults.go @@ -0,0 +1,137 @@ +package defaults + +import ( + "reflect" + "regexp" + "strconv" + "strings" + "time" +) + +// Applies the default values to the struct object, the struct type must have +// the StructTag with name "default" and the directed value. +// +// Usage +// type ExampleBasic struct { +// Foo bool `default:"true"` +// Bar string `default:"33"` +// Qux int8 +// Dur time.Duration `default:"2m3s"` +// } +// +// foo := &ExampleBasic{} +// SetDefaults(foo) +func SetDefaults(variable interface{}) { + getDefaultFiller().Fill(variable) +} + +var defaultFiller *Filler = nil + +func getDefaultFiller() *Filler { + if defaultFiller == nil { + defaultFiller = newDefaultFiller() + } + + return defaultFiller +} + +func newDefaultFiller() *Filler { + funcs := make(map[reflect.Kind]FillerFunc, 0) + funcs[reflect.Bool] = func(field *FieldData) { + value, _ := strconv.ParseBool(field.TagValue) + field.Value.SetBool(value) + } + + funcs[reflect.Int] = func(field *FieldData) { + value, _ := strconv.ParseInt(field.TagValue, 10, 64) + field.Value.SetInt(value) + } + + funcs[reflect.Int8] = funcs[reflect.Int] + funcs[reflect.Int16] = funcs[reflect.Int] + funcs[reflect.Int32] = funcs[reflect.Int] + funcs[reflect.Int64] = func(field *FieldData) { + if field.Field.Type == reflect.TypeOf(time.Second) { + value, _ := time.ParseDuration(field.TagValue) + field.Value.Set(reflect.ValueOf(value)) + } else { + value, _ := strconv.ParseInt(field.TagValue, 10, 64) + field.Value.SetInt(value) + } + } + + funcs[reflect.Float32] = func(field *FieldData) { + value, _ := strconv.ParseFloat(field.TagValue, 64) + field.Value.SetFloat(value) + } + + funcs[reflect.Float64] = funcs[reflect.Float32] + + funcs[reflect.Uint] = func(field *FieldData) { + value, _ := strconv.ParseUint(field.TagValue, 10, 64) + field.Value.SetUint(value) + } + + funcs[reflect.Uint8] = funcs[reflect.Uint] + funcs[reflect.Uint16] = funcs[reflect.Uint] + funcs[reflect.Uint32] = funcs[reflect.Uint] + funcs[reflect.Uint64] = funcs[reflect.Uint] + + funcs[reflect.String] = func(field *FieldData) { + field.Value.SetString(field.TagValue) + } + + funcs[reflect.Struct] = func(field *FieldData) { + fields := getDefaultFiller().GetFieldsFromValue(field.Value, nil) + getDefaultFiller().SetDefaultValues(fields) + } + + types := make(map[TypeHash]FillerFunc, 1) + types["time.Duration"] = func(field *FieldData) { + d, _ := time.ParseDuration(field.TagValue) + field.Value.Set(reflect.ValueOf(d)) + } + + funcs[reflect.Slice] = func(field *FieldData) { + k := field.Value.Type().Elem().Kind() + switch k { + case reflect.Uint8: + if field.Value.Bytes() != nil { + return + } + field.Value.SetBytes([]byte(field.TagValue)) + case reflect.Struct: + count := field.Value.Len() + for i := 0; i < count; i++ { + fields := getDefaultFiller().GetFieldsFromValue(field.Value.Index(i), nil) + getDefaultFiller().SetDefaultValues(fields) + } + default: + //处理形如 [1,2,3,4] + reg := regexp.MustCompile(`^\[(.*)\]$`) + matchs := reg.FindStringSubmatch(field.TagValue) + if len(matchs) != 2 { + return + } + if matchs[1] == "" { + field.Value.Set(reflect.MakeSlice(field.Value.Type(), 0, 0)) + } else { + defaultValue := strings.Split(matchs[1], ",") + result := reflect.MakeSlice(field.Value.Type(), len(defaultValue), len(defaultValue)) + for i := 0; i < len(defaultValue); i++ { + itemValue := result.Index(i) + item := &FieldData{ + Value: itemValue, + Field: reflect.StructField{}, + TagValue: defaultValue[i], + Parent: nil, + } + funcs[k](item) + } + field.Value.Set(result) + } + } + } + + return &Filler{FuncByKind: funcs, FuncByType: types, Tag: "default"} +} diff --git a/vendor/github.com/mcuadros/go-defaults/doc.go b/vendor/github.com/mcuadros/go-defaults/doc.go new file mode 100644 index 000000000..373764695 --- /dev/null +++ b/vendor/github.com/mcuadros/go-defaults/doc.go @@ -0,0 +1,5 @@ +/* +This library allow to define a default value to any struct, this is made thanks +to struct tags. +*/ +package defaults diff --git a/vendor/github.com/mcuadros/go-defaults/factory.go b/vendor/github.com/mcuadros/go-defaults/factory.go new file mode 100644 index 000000000..99f9d2755 --- /dev/null +++ b/vendor/github.com/mcuadros/go-defaults/factory.go @@ -0,0 +1,87 @@ +package defaults + +import ( + "crypto/md5" + "encoding/hex" + "math/rand" + "reflect" + "time" +) + +func Factory(variable interface{}) { + getFactoryFiller().Fill(variable) +} + +var factoryFiller *Filler = nil + +func getFactoryFiller() *Filler { + if factoryFiller == nil { + factoryFiller = newFactoryFiller() + } + + return factoryFiller +} + +func newFactoryFiller() *Filler { + rand.Seed(time.Now().UTC().UnixNano()) + + funcs := make(map[reflect.Kind]FillerFunc, 0) + + funcs[reflect.Bool] = func(field *FieldData) { + if rand.Intn(1) == 1 { + field.Value.SetBool(true) + } else { + field.Value.SetBool(false) + } + } + + funcs[reflect.Int] = func(field *FieldData) { + field.Value.SetInt(int64(rand.Int())) + } + + funcs[reflect.Int8] = funcs[reflect.Int] + funcs[reflect.Int16] = funcs[reflect.Int] + funcs[reflect.Int32] = funcs[reflect.Int] + funcs[reflect.Int64] = funcs[reflect.Int] + + funcs[reflect.Float32] = func(field *FieldData) { + field.Value.SetFloat(rand.Float64()) + } + + funcs[reflect.Float64] = funcs[reflect.Float32] + + funcs[reflect.Uint] = func(field *FieldData) { + field.Value.SetUint(uint64(rand.Uint32())) + } + + funcs[reflect.Uint8] = funcs[reflect.Uint] + funcs[reflect.Uint16] = funcs[reflect.Uint] + funcs[reflect.Uint32] = funcs[reflect.Uint] + funcs[reflect.Uint64] = funcs[reflect.Uint] + + funcs[reflect.String] = func(field *FieldData) { + field.Value.SetString(randomString()) + } + + funcs[reflect.Slice] = func(field *FieldData) { + if field.Value.Type().Elem().Kind() == reflect.Uint8 { + if field.Value.Bytes() != nil { + return + } + + field.Value.SetBytes([]byte(randomString())) + } + } + + funcs[reflect.Struct] = func(field *FieldData) { + fields := getFactoryFiller().GetFieldsFromValue(field.Value, nil) + getFactoryFiller().SetDefaultValues(fields) + } + + return &Filler{FuncByKind: funcs, Tag: "factory"} +} + +func randomString() string { + hash := md5.Sum([]byte(time.Now().UTC().String())) + return hex.EncodeToString(hash[:]) +} diff --git a/vendor/github.com/mcuadros/go-defaults/filler.go b/vendor/github.com/mcuadros/go-defaults/filler.go new file mode 100644 index 000000000..abacefadb --- /dev/null +++ b/vendor/github.com/mcuadros/go-defaults/filler.go @@ -0,0 +1,147 @@ +package defaults + +import ( + "fmt" + "reflect" +) + +type FieldData struct { + Field reflect.StructField + Value reflect.Value + TagValue string + Parent *FieldData +} + +type FillerFunc func(field *FieldData) + +// Filler contains all the functions to fill any struct field with any type +// allowing to define function by Kind, Type of field name +type Filler struct { + FuncByName map[string]FillerFunc + FuncByType map[TypeHash]FillerFunc + FuncByKind map[reflect.Kind]FillerFunc + Tag string +} + +// Fill apply all the functions contained on Filler, setting all the possible +// values +func (f *Filler) Fill(variable interface{}) { + fields := f.getFields(variable) + f.SetDefaultValues(fields) +} + +func (f *Filler) getFields(variable interface{}) []*FieldData { + valueObject := reflect.ValueOf(variable).Elem() + + return f.GetFieldsFromValue(valueObject, nil) +} + +func (f *Filler) GetFieldsFromValue(valueObject reflect.Value, parent *FieldData) []*FieldData { + typeObject := valueObject.Type() + + count := valueObject.NumField() + var results []*FieldData + for i := 0; i < count; i++ { + value := valueObject.Field(i) + field := typeObject.Field(i) + + if value.CanSet() { + results = append(results, &FieldData{ + Value: value, + Field: field, + TagValue: field.Tag.Get(f.Tag), + Parent: parent, + }) + } + } + + return results +} + +func (f *Filler) SetDefaultValues(fields []*FieldData) { + for _, field := range fields { + if f.isEmpty(field) { + f.SetDefaultValue(field) + } + } +} + +func (f *Filler) isEmpty(field *FieldData) bool { + switch field.Value.Kind() { + case reflect.Bool: + return !field.Value.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return field.Value.Int() == 0 + case reflect.Float32, reflect.Float64: + return field.Value.Float() == .0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return field.Value.Uint() == 0 + case reflect.Slice: + switch field.Value.Type().Elem().Kind() { + case reflect.Struct: + // always assume the structs in the slice is empty and can be filled + // the actually struct filling logic should take care of the rest + return true + default: + return field.Value.Len() == 0 + } + case reflect.String: + return field.Value.String() == "" + } + return true +} + +func (f *Filler) SetDefaultValue(field *FieldData) { + getters := []func(field *FieldData) FillerFunc{ + f.getFunctionByName, + f.getFunctionByType, + f.getFunctionByKind, + } + + for _, getter := range getters { + filler := getter(field) + if filler != nil { + filler(field) + return + } + } + + return +} + +func (f *Filler) getFunctionByName(field *FieldData) FillerFunc { + if f, ok := f.FuncByName[field.Field.Name]; ok == true { + return f + } + + return nil +} + +func (f *Filler) getFunctionByType(field *FieldData) FillerFunc { + if f, ok := f.FuncByType[GetTypeHash(field.Field.Type)]; ok == true { + return f + } + + return nil +} + +func (f *Filler) getFunctionByKind(field *FieldData) FillerFunc { + if f, ok := f.FuncByKind[field.Field.Type.Kind()]; ok == true { + return f + } + + return nil +} + +// TypeHash is a string representing a reflect.Type following the next pattern: +// . +type TypeHash string + +// GetTypeHash returns the TypeHash for a given reflect.Type +func GetTypeHash(t reflect.Type) TypeHash { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + + return TypeHash(fmt.Sprintf("%s.%s", t.PkgPath(), t.Name())) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b6c6b3290..98a2adc52 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -809,6 +809,9 @@ github.com/mattn/go-colorable # github.com/mattn/go-isatty v0.0.20 ## explicit; go 1.15 github.com/mattn/go-isatty +# github.com/mcuadros/go-defaults v1.2.0 +## explicit; go 1.14 +github.com/mcuadros/go-defaults # github.com/miekg/dns v1.1.62 ## explicit; go 1.19 github.com/miekg/dns