From f6c4b245ad74bc9fb93c6d6fc4c2343e88ab531a Mon Sep 17 00:00:00 2001 From: Matt Cotter Date: Wed, 11 Jun 2025 15:26:07 -0500 Subject: [PATCH] test: move snapshot integration tests to unit test --- integration/scripts/test_snapshot_conf.py | 93 ------------------- integration/tests/integration.tftest.hcl | 25 ----- internal/commands/config/config_test.go | 85 ++++++++++++++++- .../config/test/snap1-docker-output.yaml | 0 .../config/test/snap1-full-agent-config.yaml | 0 .../config/test/snap1-linux-output.yaml | 0 .../config/test/snap1-windows-output.yaml | 0 ...fig.yaml => snap2-empty-agent-config.yaml} | 0 ...tel-config.yaml => snap2-otel-config.yaml} | 0 ...utput.yaml => snap2-with-otel-output.yaml} | 0 internal/connections/confighandler.go | 52 ----------- internal/root/root.go | 53 +++++++++++ 12 files changed, 133 insertions(+), 175 deletions(-) delete mode 100755 integration/scripts/test_snapshot_conf.py rename integration/scripts/snapshots/docker.yaml => internal/commands/config/test/snap1-docker-output.yaml (100%) rename integration/scripts/snapshots/full-agent-config.yaml => internal/commands/config/test/snap1-full-agent-config.yaml (100%) rename integration/scripts/snapshots/linux.yaml => internal/commands/config/test/snap1-linux-output.yaml (100%) rename integration/scripts/snapshots/windows.yaml => internal/commands/config/test/snap1-windows-output.yaml (100%) rename internal/commands/config/test/{agent-config.yaml => snap2-empty-agent-config.yaml} (100%) rename internal/commands/config/test/{otel-config.yaml => snap2-otel-config.yaml} (100%) rename internal/commands/config/test/{output.yaml => snap2-with-otel-output.yaml} (100%) diff --git a/integration/scripts/test_snapshot_conf.py b/integration/scripts/test_snapshot_conf.py deleted file mode 100755 index 7ed5fc550..000000000 --- a/integration/scripts/test_snapshot_conf.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -import utils as u -import os -import difflib - -AGENT_CONFIG_FILE_NAME = "full-agent-config.yaml" - - -def upload_agent_config(remote_host: u.Host, remote_path: str) -> None: - current_script_dir = os.path.dirname(os.path.abspath(__file__)) - # Upload the full observe-agent config to remote host - local_path = os.path.abspath( - os.path.join(current_script_dir, "snapshots", AGENT_CONFIG_FILE_NAME) - ) - print(f"Path to '{AGENT_CONFIG_FILE_NAME}' file: {local_path}") - remote_host.put_file(local_path=local_path, remote_path=remote_path) - - -def do_snapshot_test( - remote_host: u.Host, remote_path: str, config_command: str, snapshot_file: str -) -> None: - # Copy the agent config to the remote host - upload_agent_config(remote_host, remote_path) - result = remote_host.run_command(config_command) - if result.exited != 0 or result.stderr: - u.print_remote_result(result) - raise ValueError(f"❌ Error in rendering config for {snapshot_file}") - rendered_config = result.stdout - current_script_dir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(current_script_dir, "snapshots", snapshot_file), "r") as f: - expected_config = f.read() - if rendered_config == expected_config: - print(f" ✅ Config match for {snapshot_file}") - return - diff = difflib.Differ().compare( - expected_config.splitlines(), rendered_config.splitlines() - ) - print(f"Diff expected vs actual for {snapshot_file}:") - print("\n".join(diff)) - raise ValueError(f"❌ Config mismatch for {snapshot_file}") - - -@u.print_test_decorator -def run_test_windows(remote_host: u.Host, env_vars: dict) -> None: - user = env_vars["user"] - remote_path = f"/C:/Users/{user}" # for user in sftp - config_command = f"Set-Location \"C:\\Program Files\\Observe\\observe-agent\"; ./observe-agent --observe-config C:\\Users\\{user}\\{AGENT_CONFIG_FILE_NAME} config --render-otel" - snapshot_file = "windows.yaml" - do_snapshot_test(remote_host, remote_path, config_command, snapshot_file) - - -@u.print_test_decorator -def run_test_docker(remote_host: u.Host, env_vars: dict) -> None: - docker_prefix = u.get_docker_prefix( - remote_host, - False, - extra_args=f"--mount type=bind,source=$(pwd)/{AGENT_CONFIG_FILE_NAME},target=/etc/observe-agent/{AGENT_CONFIG_FILE_NAME}", - ) - remote_path = "/home/" + env_vars["user"] - config_command = f"{docker_prefix} --observe-config /etc/observe-agent/{AGENT_CONFIG_FILE_NAME} config --render-otel" - snapshot_file = "docker.yaml" - do_snapshot_test(remote_host, remote_path, config_command, snapshot_file) - - -@u.print_test_decorator -def run_test_linux(remote_host: u.Host, env_vars: dict) -> None: - remote_path = "/home/" + env_vars["user"] - config_command = f"observe-agent --observe-config {remote_path}/{AGENT_CONFIG_FILE_NAME} config --render-otel" - snapshot_file = "linux.yaml" - do_snapshot_test(remote_host, remote_path, config_command, snapshot_file) - - -if __name__ == "__main__": - env_vars = u.get_env_vars() - remote_host = u.Host( - host_ip=env_vars["host"], - username=env_vars["user"], - key_file_path=env_vars["key_filename"], - password=env_vars["password"], - ) - - # Test SSH Connection before starting test of interest - remote_host.test_conection(int(env_vars["machine_config"]["sleep"])) - - if ( - "redhat" in env_vars["machine_config"]["distribution"] - or "debian" in env_vars["machine_config"]["distribution"] - ): - run_test_linux(remote_host, env_vars) - elif "windows" in env_vars["machine_config"]["distribution"]: - run_test_windows(remote_host, env_vars) - elif "docker" in env_vars["machine_config"]["distribution"]: - run_test_docker(remote_host, env_vars) diff --git a/integration/tests/integration.tftest.hcl b/integration/tests/integration.tftest.hcl index 1c3607684..2a51e226a 100644 --- a/integration/tests/integration.tftest.hcl +++ b/integration/tests/integration.tftest.hcl @@ -89,31 +89,6 @@ run "test_version" { } } -run "test_snapshot_conf" { - module { - source = "observeinc/collection/aws//modules/testing/exec" - version = "2.9.0" - } - - variables { - command = "python3 ./scripts/test_snapshot_conf.py" - env_vars = { - HOST = run.setup_ec2.public_ip - USER = run.setup_ec2.user_name - KEY_FILENAME = run.setup_ec2.private_key_path - PASSWORD = run.setup_ec2.password - MACHINE_NAME = run.setup_ec2.machine_name - MACHINE_CONFIG = run.setup_ec2.machine_config - } - } - - assert { - condition = output.error == "" - error_message = "Error in Snapshot Test" - } -} - - run "test_configure" { module { source = "observeinc/collection/aws//modules/testing/exec" diff --git a/internal/commands/config/config_test.go b/internal/commands/config/config_test.go index 923c728d0..4348a9da8 100644 --- a/internal/commands/config/config_test.go +++ b/internal/commands/config/config_test.go @@ -22,8 +22,53 @@ import ( "github.com/stretchr/testify/assert" ) -// TODO rework this test to handle our snapshot tests as go unit tests. +type PackageType string + +const MacOS = PackageType("macos") +const Linux = PackageType("linux") +const Windows = PackageType("windows") +const Docker = PackageType("docker") + +type snapshotTest struct { + agentConfigPath string + otelConfigPath string + outputPath string + packageType PackageType +} + +var allSnapshotTests = []snapshotTest{ + { + agentConfigPath: "test/snap1-full-agent-config.yaml", + outputPath: "test/snap1-docker-output.yaml", + packageType: Docker, + }, + { + agentConfigPath: "test/snap1-full-agent-config.yaml", + outputPath: "test/snap1-linux-output.yaml", + packageType: Linux, + }, + { + agentConfigPath: "test/snap1-full-agent-config.yaml", + outputPath: "test/snap1-windows-output.yaml", + packageType: Windows, + }, + { + agentConfigPath: "test/snap2-empty-agent-config.yaml", + otelConfigPath: "test/snap2-otel-config.yaml", + outputPath: "test/snap2-with-otel-output.yaml", + packageType: MacOS, + }, +} + func Test_RenderOtelConfig(t *testing.T) { + for _, test := range allSnapshotTests { + t.Run(test.outputPath, func(t *testing.T) { + runSnapshotTest(t, test) + }) + } +} + +func runSnapshotTest(t *testing.T, test snapshotTest) { // Get current path _, filename, _, ok := runtime.Caller(0) assert.True(t, ok) @@ -31,22 +76,52 @@ func Test_RenderOtelConfig(t *testing.T) { // Set the template base dir for all connections for _, conn := range connections.AllConnectionTypes { - conn.ApplyOptions(connections.WithConfigFolderPath(filepath.Join(curPath, "../../../packaging/macos/connections"))) + conn.ApplyOptions(connections.WithConfigFolderPath(getPackagingPath(t, test.packageType, curPath))) } // Set config flags flags := pflag.NewFlagSet("test", pflag.ContinueOnError) observecol.AddConfigFlags(flags) - flags.Parse([]string{"--config", filepath.Join(curPath, "test/otel-config.yaml")}) + if test.otelConfigPath != "" { + flags.Parse([]string{"--config", filepath.Join(curPath, test.otelConfigPath)}) + } viper.Reset() - root.CfgFile = filepath.Join(curPath, "test/agent-config.yaml") + root.CfgFile = filepath.Join(curPath, test.agentConfigPath) root.InitConfig() + setEnvVars(t, test.packageType) // Run the test ctx := logger.WithCtx(context.Background(), logger.GetNop()) var output bytes.Buffer PrintShortOtelConfig(ctx, &output) - expected, err := os.ReadFile(filepath.Join(curPath, "test/output.yaml")) + expected, err := os.ReadFile(filepath.Join(curPath, test.outputPath)) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(string(expected)), strings.TrimSpace(output.String())) } + +func getPackagingPath(t *testing.T, packageType PackageType, curPath string) string { + const packagingPath = "../../../packaging" + switch packageType { + case MacOS, Linux, Windows: + return filepath.Join(curPath, packagingPath, string(packageType), "connections") + case Docker: + return filepath.Join(curPath, packagingPath, "docker/observe-agent/connections") + default: + t.Errorf("Unknown package type: %s", packageType) + return "" + } +} + +func setEnvVars(t *testing.T, packageType PackageType) { + switch packageType { + case MacOS: + assert.NoError(t, os.Setenv("FILESTORAGE_PATH", "/var/lib/observe-agent/filestorage")) + case Windows: + assert.NoError(t, os.Setenv("FILESTORAGE_PATH", "C:\\ProgramData\\Observe\\observe-agent\\filestorage")) + case Linux, Docker: + assert.NoError(t, os.Setenv("FILESTORAGE_PATH", "/var/lib/observe-agent/filestorage")) + default: + t.Errorf("Unknown package type: %s", packageType) + } + +} diff --git a/integration/scripts/snapshots/docker.yaml b/internal/commands/config/test/snap1-docker-output.yaml similarity index 100% rename from integration/scripts/snapshots/docker.yaml rename to internal/commands/config/test/snap1-docker-output.yaml diff --git a/integration/scripts/snapshots/full-agent-config.yaml b/internal/commands/config/test/snap1-full-agent-config.yaml similarity index 100% rename from integration/scripts/snapshots/full-agent-config.yaml rename to internal/commands/config/test/snap1-full-agent-config.yaml diff --git a/integration/scripts/snapshots/linux.yaml b/internal/commands/config/test/snap1-linux-output.yaml similarity index 100% rename from integration/scripts/snapshots/linux.yaml rename to internal/commands/config/test/snap1-linux-output.yaml diff --git a/integration/scripts/snapshots/windows.yaml b/internal/commands/config/test/snap1-windows-output.yaml similarity index 100% rename from integration/scripts/snapshots/windows.yaml rename to internal/commands/config/test/snap1-windows-output.yaml diff --git a/internal/commands/config/test/agent-config.yaml b/internal/commands/config/test/snap2-empty-agent-config.yaml similarity index 100% rename from internal/commands/config/test/agent-config.yaml rename to internal/commands/config/test/snap2-empty-agent-config.yaml diff --git a/internal/commands/config/test/otel-config.yaml b/internal/commands/config/test/snap2-otel-config.yaml similarity index 100% rename from internal/commands/config/test/otel-config.yaml rename to internal/commands/config/test/snap2-otel-config.yaml diff --git a/internal/commands/config/test/output.yaml b/internal/commands/config/test/snap2-with-otel-output.yaml similarity index 100% rename from internal/commands/config/test/output.yaml rename to internal/commands/config/test/snap2-with-otel-output.yaml diff --git a/internal/connections/confighandler.go b/internal/connections/confighandler.go index 32f0a4800..da74bd2d9 100644 --- a/internal/connections/confighandler.go +++ b/internal/connections/confighandler.go @@ -6,9 +6,7 @@ import ( "os" "path/filepath" "runtime" - "strings" - "github.com/observeinc/observe-agent/internal/commands/util" "github.com/observeinc/observe-agent/internal/commands/util/logger" "github.com/observeinc/observe-agent/internal/config" "github.com/spf13/viper" @@ -20,11 +18,6 @@ const ( ) func SetupAndGetConfigFiles(ctx context.Context) ([]string, func(), error) { - // Set Env Vars from config - err := SetEnvVars() - if err != nil { - return nil, nil, err - } // Set up our temp dir annd temp config files tmpDir, err := os.MkdirTemp("", TempFilesFolder) if err != nil { @@ -84,38 +77,6 @@ func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, er return configFilePaths, nil } -func SetEnvVars() error { - collector_url, token, debug := viper.GetString("observe_url"), viper.GetString("token"), viper.GetBool("debug") - // Ensure the collector url does not end with a slash for consistency. This will allow endpoints to be configured like: - // "${env:OBSERVE_COLLECTOR_URL}/v1/kubernetes/v1/entity" - // without worrying about a double slash. - collector_url = strings.TrimRight(collector_url, "/") - otelEndpoint := util.JoinUrl(collector_url, "/v2/otel") - promEndpoint := util.JoinUrl(collector_url, "/v1/prometheus") - // Setting values from the Observe agent config as env vars to fill in the OTEL collector config - os.Setenv("OBSERVE_COLLECTOR_URL", collector_url) - os.Setenv("OBSERVE_OTEL_ENDPOINT", otelEndpoint) - os.Setenv("OBSERVE_PROMETHEUS_ENDPOINT", promEndpoint) - os.Setenv("OBSERVE_AUTHORIZATION_HEADER", "Bearer "+token) - os.Setenv("FILESTORAGE_PATH", GetDefaultFilestoragePath()) - - // Default TRACE_TOKEN to be the value of the configured token if it's not set. This allows for users to upgrade to - // direct write tracing with ingest tokens in kubernetes without breaking backwards compatibility in our helm chart. - // TODO: remove this once our helm chart no longer supports TRACE_TOKEN - if os.Getenv("TRACE_TOKEN") == "" { - os.Setenv("TRACE_TOKEN", token) - } - - if os.Getenv("OTEL_LOG_LEVEL") == "" { - if debug { - os.Setenv("OTEL_LOG_LEVEL", "DEBUG") - } else { - os.Setenv("OTEL_LOG_LEVEL", "INFO") - } - } - return nil -} - func GetOverrideConfigFile(tmpDir string, data map[string]any) (string, error) { f, err := os.CreateTemp(tmpDir, "otel-config-overrides-*.yaml") if err != nil { @@ -148,16 +109,3 @@ func GetDefaultAgentPath() string { return "/etc/observe-agent" } } - -func GetDefaultFilestoragePath() string { - switch currOS := runtime.GOOS; currOS { - case "darwin": - return "/var/lib/observe-agent/filestorage" - case "windows": - return os.ExpandEnv("$ProgramData\\Observe\\observe-agent\\filestorage") - case "linux": - return "/var/lib/observe-agent/filestorage" - default: - return "/var/lib/observe-agent/filestorage" - } -} diff --git a/internal/root/root.go b/internal/root/root.go index ee16693cd..7c40fe57f 100644 --- a/internal/root/root.go +++ b/internal/root/root.go @@ -7,8 +7,11 @@ import ( "context" "fmt" "os" + "runtime" + "strings" "github.com/observeinc/observe-agent/build" + "github.com/observeinc/observe-agent/internal/commands/util" "github.com/observeinc/observe-agent/internal/commands/util/logger" "github.com/observeinc/observe-agent/internal/config" "github.com/observeinc/observe-agent/internal/connections" @@ -79,4 +82,54 @@ func InitConfig() { // Apply feature gates observecol.ApplyFeatureGates(ctx) + + // Set up env vars + if err := setEnvVars(); err != nil { + fmt.Fprintln(os.Stderr, "error setting env vars:", err) + } +} + +func setEnvVars() error { + collector_url, token, debug := viper.GetString("observe_url"), viper.GetString("token"), viper.GetBool("debug") + // Ensure the collector url does not end with a slash for consistency. This will allow endpoints to be configured like: + // "${env:OBSERVE_COLLECTOR_URL}/v1/kubernetes/v1/entity" + // without worrying about a double slash. + collector_url = strings.TrimRight(collector_url, "/") + otelEndpoint := util.JoinUrl(collector_url, "/v2/otel") + promEndpoint := util.JoinUrl(collector_url, "/v1/prometheus") + // Setting values from the Observe agent config as env vars to fill in the OTEL collector config + os.Setenv("OBSERVE_COLLECTOR_URL", collector_url) + os.Setenv("OBSERVE_OTEL_ENDPOINT", otelEndpoint) + os.Setenv("OBSERVE_PROMETHEUS_ENDPOINT", promEndpoint) + os.Setenv("OBSERVE_AUTHORIZATION_HEADER", "Bearer "+token) + os.Setenv("FILESTORAGE_PATH", getDefaultFilestoragePath()) + + // Default TRACE_TOKEN to be the value of the configured token if it's not set. This allows for users to upgrade to + // direct write tracing with ingest tokens in kubernetes without breaking backwards compatibility in our helm chart. + // TODO: remove this once our helm chart no longer supports TRACE_TOKEN + if os.Getenv("TRACE_TOKEN") == "" { + os.Setenv("TRACE_TOKEN", token) + } + + if os.Getenv("OTEL_LOG_LEVEL") == "" { + if debug { + os.Setenv("OTEL_LOG_LEVEL", "DEBUG") + } else { + os.Setenv("OTEL_LOG_LEVEL", "INFO") + } + } + return nil +} + +func getDefaultFilestoragePath() string { + switch currOS := runtime.GOOS; currOS { + case "darwin": + return "/var/lib/observe-agent/filestorage" + case "windows": + return os.ExpandEnv("$ProgramData\\Observe\\observe-agent\\filestorage") + case "linux": + return "/var/lib/observe-agent/filestorage" + default: + return "/var/lib/observe-agent/filestorage" + } }