refactor to use agent config

Author: obs-gh-mattcotter

internal/commands/config/config.go

@@ -6,20 +6,12 @@ package config
 import (
 	"context"
 	"fmt"
-	"os"
-	"strings"
 
-	"github.com/go-viper/mapstructure/v2"
 	"github.com/observeinc/observe-agent/internal/commands/start"
-	logger "github.com/observeinc/observe-agent/internal/commands/util"
-	"github.com/observeinc/observe-agent/internal/config"
+	"github.com/observeinc/observe-agent/internal/commands/util"
+	"github.com/observeinc/observe-agent/internal/commands/util/logger"
 	"github.com/observeinc/observe-agent/internal/root"
-	"github.com/observeinc/observe-agent/observecol"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"go.opentelemetry.io/collector/confmap"
-	"go.opentelemetry.io/collector/otelcol"
-	"gopkg.in/yaml.v3"
 )
 
 var configCmd = &cobra.Command{
@@ -49,96 +41,14 @@ bundled OTel configuration.`,
 			return err
 		}
 		if singleOtel {
-			return printShortOtelConfig(ctx, configFilePaths)
+			return util.PrintShortOtelConfig(ctx, configFilePaths)
 		} else if detailedOtel {
-			return printFullOtelConfig(configFilePaths)
+			return util.PrintFullOtelConfig(configFilePaths)
 		}
-		return printAllConfigsIndividually(configFilePaths)
+		return util.PrintAllConfigsIndividually(configFilePaths)
 	},
 }
 
-func printAllConfigsIndividually(configFilePaths []string) error {
-	printConfig := func(comment string, data []byte) {
-		fmt.Printf("# ======== %s\n", comment)
-		fmt.Println(strings.Trim(string(data), "\n\t "))
-		fmt.Println("---")
-	}
-
-	agentConfig, err := config.AgentConfigFromViper(viper.GetViper())
-	if err != nil {
-		return err
-	}
-	agentConfigYaml, err := yaml.Marshal(agentConfig)
-	if err != nil {
-		return err
-	}
-	printConfig("computed agent config", agentConfigYaml)
-	agentConfigFile := viper.ConfigFileUsed()
-	if agentConfigFile != "" {
-		configFilePaths = append([]string{agentConfigFile}, configFilePaths...)
-	}
-	for _, filePath := range configFilePaths {
-		file, err := os.ReadFile(filePath)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "error reading config file %s: %s", filePath, err.Error())
-		} else {
-			printConfig("config file "+filePath, file)
-		}
-	}
-	return nil
-}
-
-func printShortOtelConfig(ctx context.Context, configFilePaths []string) error {
-	if len(configFilePaths) == 0 {
-		return nil
-	}
-	settings := observecol.ConfigProviderSettings(configFilePaths)
-	resolver, err := confmap.NewResolver(settings.ResolverSettings)
-	if err != nil {
-		return fmt.Errorf("failed to create new resolver: %w", err)
-	}
-	conf, err := resolver.Resolve(ctx)
-	if err != nil {
-		return fmt.Errorf("error while resolving config: %w", err)
-	}
-	b, err := yaml.Marshal(conf.ToStringMap())
-	if err != nil {
-		return fmt.Errorf("error while marshaling to YAML: %w", err)
-	}
-	fmt.Printf("%s\n", b)
-	return nil
-}
-
-func printFullOtelConfig(configFilePaths []string) error {
-	if len(configFilePaths) == 0 {
-		return nil
-	}
-	colSettings := observecol.GenerateCollectorSettingsWithConfigFiles(configFilePaths)
-	factories, err := colSettings.Factories()
-	if err != nil {
-		return fmt.Errorf("failed to create component factory maps: %w", err)
-	}
-	provider, err := otelcol.NewConfigProvider(colSettings.ConfigProviderSettings)
-	if err != nil {
-		return fmt.Errorf("failed to create config provider: %w", err)
-	}
-	cfg, err := provider.Get(context.Background(), factories)
-	if err != nil {
-		return fmt.Errorf("failed to get config: %w", err)
-	}
-	var cfgMap map[string]any
-	err = mapstructure.Decode(cfg, &cfgMap)
-	if err != nil {
-		return fmt.Errorf("failed to marshall config to map: %w", err)
-	}
-	cfgYaml, err := yaml.Marshal(cfgMap)
-	if err != nil {
-		return fmt.Errorf("failed to marshall config to yaml: %w", err)
-	}
-	fmt.Printf("%s\n", cfgYaml)
-	return nil
-}
-
 func init() {
 	configCmd.Flags().Bool("render-otel-details", false, "Print the full resolved otel configuration including default values after the otel components perform their semantic processing.")
 	configCmd.Flags().Bool("render-otel", false, "Print a single rendered otel configuration file. This file is equivalent to the bundled configuration enabled in the observe-agent config.")

internal/commands/diagnose/agentstatuscheck.go

@@ -4,6 +4,7 @@ import (
 	"embed"
 
 	"github.com/observeinc/observe-agent/internal/commands/status"
+	"github.com/observeinc/observe-agent/internal/config"
 	"github.com/spf13/viper"
 )
 
@@ -14,7 +15,11 @@ type StatusTestResult struct {
 }
 
 func checkStatus(v *viper.Viper) (bool, any, error) {
-	data, err := status.GetStatusData(v.GetString(status.TelemetryEndpointFlag), v.GetString(status.HealthcheckEndpointFlag), v.GetString(status.HealthcheckPathFlag))
+	conf, err := config.AgentConfigFromViper(v)
+	if err != nil {
+		return false, StatusTestResult{Error: err.Error()}, err
+	}
+	data, err := status.GetStatusData(conf)
 	if err != nil {
 		return false, StatusTestResult{
 			Passed:       false,

internal/commands/diagnose/diagnose.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"text/template"
 
-	"github.com/observeinc/observe-agent/internal/commands/status"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -72,6 +71,5 @@ to attempt to identify issues that could cause the agent to function improperly.
 func init() {
 	v := viper.GetViper()
 	diagnoseCmd := NewDiagnoseCmd(v)
-	status.RegisterStatusFlags(diagnoseCmd, v)
 	root.RootCmd.AddCommand(diagnoseCmd)
 }

internal/commands/diagnose/otelconfigcheck.go

@@ -5,7 +5,7 @@ import (
 	"embed"
 
 	"github.com/observeinc/observe-agent/internal/commands/start"
-	logger "github.com/observeinc/observe-agent/internal/commands/util"
+	"github.com/observeinc/observe-agent/internal/commands/util/logger"
 	"github.com/observeinc/observe-agent/observecol"
 	"github.com/spf13/viper"
 	"go.opentelemetry.io/collector/otelcol"

internal/commands/start/start.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"strings"
 
-	logger "github.com/observeinc/observe-agent/internal/commands/util"
+	"github.com/observeinc/observe-agent/internal/commands/util/logger"
 	"github.com/observeinc/observe-agent/internal/connections"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/observeinc/observe-agent/observecol"

internal/commands/status/flags.go

@@ -8,6 +8,7 @@ import (
 	"html/template"
 	"os"
 
+	"github.com/observeinc/observe-agent/internal/config"
 	"github.com/observeinc/observe-agent/internal/connections"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/spf13/cobra"
@@ -35,12 +36,15 @@ func NewStatusCmd(v *viper.Viper) *cobra.Command {
 func init() {
 	v := viper.GetViper()
 	statusCmd := NewStatusCmd(v)
-	RegisterStatusFlags(statusCmd, v)
 	root.RootCmd.AddCommand(statusCmd)
 }
 
 func getStatusFromTemplate(v *viper.Viper) error {
-	data, err := GetStatusData(v.GetString(TelemetryEndpointFlag), v.GetString(HealthcheckEndpointFlag), v.GetString(HealthcheckPathFlag))
+	conf, err := config.AgentConfigFromViper(v)
+	if err != nil {
+		return err
+	}
+	data, err := GetStatusData(conf)
 	if err != nil {
 		return err
 	}

internal/commands/status/status.go

@@ -4,11 +4,13 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"net/url"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/observeinc/observe-agent/internal/commands/util"
+	"github.com/observeinc/observe-agent/internal/config"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"github.com/shirou/gopsutil/v3/host"
@@ -71,15 +73,15 @@ func bToMb(b float32) float32 {
 }
 
 func GetAgentStatusFromHealthcheck(baseURL string, path string) (AgentStatus, error) {
+	baseURL = util.ReplaceEnvString(baseURL)
+	path = util.ReplaceEnvString(path)
 	if len(path) == 0 || len(baseURL) == 0 {
 		return NotRunning, errors.New("health_check endpoint and path must be provided")
 	}
-	if baseURL[len(baseURL)-1] == '/' {
-		baseURL = baseURL[:len(baseURL)-1]
-	}
-	URL, err := url.JoinPath(baseURL, path)
-	if err != nil {
-		return NotRunning, err
+	URL := util.JoinUrl(baseURL, path)
+	// The healthcheck extension is always http
+	if !strings.Contains(URL, "://") {
+		URL = "http://" + URL
 	}
 
 	c := &http.Client{}
@@ -106,17 +108,22 @@ func getMetricsSum(metrics []*io_prometheus_client.Metric) float64 {
 	return sum
 }
 
+func GetAgentMetrics(conf *config.AgentConfig) (*AgentMetrics, error) {
+	host := util.ReplaceEnvString(conf.InternalTelemetry.Host)
+	if !strings.Contains(host, "://") {
+		host = "http://" + host
+	}
+	host = strings.TrimRight(host, ":/")
+	port := conf.InternalTelemetry.Port
+	baseURL := fmt.Sprintf("%s:%d", host, port)
+	return GetAgentMetricsFromEndpoint(baseURL)
+}
+
 func GetAgentMetricsFromEndpoint(baseURL string) (*AgentMetrics, error) {
 	if len(baseURL) == 0 {
 		return nil, errors.New("metrics endpoint must be provided")
 	}
-	if baseURL[len(baseURL)-1] == '/' {
-		baseURL = baseURL[:len(baseURL)-1]
-	}
-	URL, err := url.JoinPath(baseURL, "/metrics")
-	if err != nil {
-		return nil, err
-	}
+	URL := util.JoinUrl(baseURL, "/metrics")
 	c := &http.Client{}
 	req, err := http.NewRequest("GET", URL, nil)
 	if err != nil {
@@ -140,53 +147,53 @@ func GetAgentMetricsFromEndpoint(baseURL string) (*AgentMetrics, error) {
 		if v.Type.String() == io_prometheus_client.MetricType_HISTOGRAM.String() {
 			met := v.Metric[0]
 			switch name := *v.Name; name {
-			case "otelcol_http_client_duration":
+			case "http_client_duration_milliseconds":
 				agentMets.AvgServerResponseTime = float32(met.Histogram.GetSampleSum()) / float32(met.Histogram.GetSampleCount())
-			case "otelcol_http_server_duration":
+			case "http_server_duration_milliseconds":
 				agentMets.AvgClientResponseTime = float32(met.Histogram.GetSampleSum()) / float32(met.Histogram.GetSampleCount())
 			default:
 			}
 		} else {
 			met := v.Metric[0]
 			switch name := *v.Name; name {
 			// Log-related metrics
-			case "otelcol_receiver_accepted_log_records":
+			case "otelcol_receiver_accepted_log_records_total":
 				agentMets.LogsStats.ReceiverAcceptedCount = int(getMetricsSum(v.Metric))
-			case "otelcol_receiver_refused_log_records":
+			case "otelcol_receiver_refused_log_records_total":
 				agentMets.LogsStats.ReceiverRefusedCount = int(getMetricsSum(v.Metric))
-			case "otelcol_exporter_sent_log_records":
+			case "otelcol_exporter_sent_log_records_total":
 				agentMets.LogsStats.ExporterSentCount = int(getMetricsSum(v.Metric))
-			case "otelcol_exporter_send_failed_log_records":
+			case "otelcol_exporter_send_failed_log_records_total":
 				agentMets.LogsStats.ExporterSendFailedCount = int(getMetricsSum(v.Metric))
 
 			// Metric-related metrics
-			case "otelcol_receiver_accepted_metric_points":
+			case "otelcol_receiver_accepted_metric_points_total":
 				agentMets.MetricsStats.ReceiverAcceptedCount = int(getMetricsSum(v.Metric))
-			case "otelcol_receiver_refused_metric_points":
+			case "otelcol_receiver_refused_metric_points_total":
 				agentMets.MetricsStats.ReceiverRefusedCount = int(getMetricsSum(v.Metric))
-			case "otelcol_exporter_sent_metric_points":
+			case "otelcol_exporter_sent_metric_points_total":
 				agentMets.MetricsStats.ExporterSentCount = int(getMetricsSum(v.Metric))
-			case "otelcol_exporter_send_failed_metric_points":
+			case "otelcol_exporter_send_failed_metric_points_total":
 				agentMets.MetricsStats.ExporterSendFailedCount = int(getMetricsSum(v.Metric))
 
 			// Trace-related metrics
-			case "otelcol_receiver_accepted_spans":
+			case "otelcol_receiver_accepted_spans_total":
 				agentMets.TracesStats.ReceiverAcceptedCount = int(getMetricsSum(v.Metric))
-			case "otelcol_receiver_refused_spans":
+			case "otelcol_receiver_refused_spans_total":
 				agentMets.TracesStats.ReceiverRefusedCount = int(getMetricsSum(v.Metric))
-			case "otelcol_exporter_sent_spans":
+			case "otelcol_exporter_sent_spans_total":
 				agentMets.TracesStats.ExporterSentCount = int(getMetricsSum(v.Metric))
-			case "otelcol_exporter_send_failed_spans":
+			case "otelcol_exporter_send_failed_spans_total":
 				agentMets.TracesStats.ExporterSendFailedCount = int(getMetricsSum(v.Metric))
 
 			// General metrics
 			case "otelcol_exporter_queue_size":
 				agentMets.ExporterQueueSize = float32(met.Gauge.GetValue())
-			case "otelcol_process_cpu_seconds":
+			case "otelcol_process_cpu_seconds_total":
 				agentMets.CPUSeconds = float32(getMetricsSum(v.Metric))
-			case "otelcol_process_uptime":
+			case "otelcol_process_uptime_seconds_total":
 				agentMets.Uptime = float32(getMetricsSum(v.Metric))
-			case "otelcol_process_memory_rss":
+			case "otelcol_process_memory_rss_bytes":
 				agentMets.MemoryUsed = bToMb(float32(met.Gauge.GetValue()))
 			case "otelcol_process_runtime_total_sys_memory_bytes":
 				agentMets.TotalSysMemory = bToMb(float32(met.Gauge.GetValue()))
@@ -197,8 +204,8 @@ func GetAgentMetricsFromEndpoint(baseURL string) (*AgentMetrics, error) {
 	return &agentMets, nil
 }
 
-func GetStatusData(telemetryEndpoint string, healthcheckEndpoint string, healthcheckPath string) (*StatusData, error) {
-	agentMets, err := GetAgentMetricsFromEndpoint(telemetryEndpoint)
+func GetStatusData(conf *config.AgentConfig) (*StatusData, error) {
+	agentMets, err := GetAgentMetrics(conf)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Error getting agent metrics: ", err)
 		agentMets = &AgentMetrics{}
@@ -216,7 +223,7 @@ func GetStatusData(telemetryEndpoint string, healthcheckEndpoint string, healthc
 	if err != nil {
 		uptime = time.Duration(0)
 	}
-	status, err := GetAgentStatusFromHealthcheck(healthcheckEndpoint, healthcheckPath)
+	status, err := GetAgentStatusFromHealthcheck(conf.HealthCheck.Endpoint, conf.HealthCheck.Path)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Error receiving data from agent health check: ", err)
 		status = NotRunning