feat: enable filelog receiver in windows if include is specified

obs-gh-mattcotter · obs-gh-mattcotter · commit 5b116f2826b0 · 2025-06-12T15:07:39.000-05:00
diff --git a/internal/commands/config/config_test.go b/internal/commands/config/config_test.go
@@ -37,6 +37,7 @@ type snapshotTest struct {
 }
 
 var allSnapshotTests = []snapshotTest{
+	// Tests with full agent config
 	{
 		agentConfigPath: "test/snap1-full-agent-config.yaml",
 		outputPath:      "test/snap1-docker-output.yaml",
@@ -52,12 +53,18 @@ var allSnapshotTests = []snapshotTest{
 		outputPath:      "test/snap1-windows-output.yaml",
 		packageType:     Windows,
 	},
+	// Tests with minimal agent config
 	{
 		agentConfigPath: "test/snap2-empty-agent-config.yaml",
 		otelConfigPath:  "test/snap2-otel-config.yaml",
 		outputPath:      "test/snap2-with-otel-output.yaml",
 		packageType:     MacOS,
 	},
+	{
+		agentConfigPath: "test/snap2-empty-agent-config.yaml",
+		outputPath:      "test/snap2-windows-output.yaml",
+		packageType:     Windows,
+	},
 }
 
 func Test_RenderOtelConfig(t *testing.T) {
diff --git a/internal/commands/config/test/snap1-windows-output.yaml b/internal/commands/config/test/snap1-windows-output.yaml
@@ -115,6 +115,25 @@ receivers:
             line_end_pattern: ENDOFLINEPATTERN
         poll_interval: 5m
         start_at: beginning
+    filelog/host_monitoring:
+        exclude:
+            - exclude1
+            - exclude2
+        include:
+            - include1
+            - include2
+        include_file_path: true
+        max_log_size: 4MiB
+        operators:
+            - expr: body matches "otel-contrib"
+              type: filter
+            - combine_field: body
+              id: multiline-recombine
+              is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
+              type: recombine
+        retry_on_failure:
+            enabled: true
+        storage: file_storage
     filestats/agent:
         collection_interval: 5m
         include: C:\Program Files\Observe\observe-agent\connections\common\base.yaml.tmpl
@@ -230,6 +249,20 @@ service:
                 - batch
             receivers:
                 - otlp
+        logs/host_monitoring-file:
+            exporters:
+                - otlphttp/observe
+                - count
+            processors:
+                - memory_limiter
+                - transform/truncate
+                - resourcedetection
+                - resourcedetection/cloud
+                - attributes/observe_global_attributes
+                - resource/observe_global_resource_attributes
+                - batch
+            receivers:
+                - filelog/host_monitoring
         logs/host_monitoring-windowsevents:
             exporters:
                 - otlphttp/observe
diff --git a/internal/commands/config/test/snap2-windows-output.yaml b/internal/commands/config/test/snap2-windows-output.yaml
@@ -0,0 +1,89 @@
+connectors:
+    count: null
+exporters:
+    debug: null
+    nop: null
+    otlphttp/observe:
+        compression: zstd
+        endpoint: https://test.collect.observeinc.com/v2/otel
+        headers:
+            authorization: Bearer 12345678901234567890:abcdefghijklmnopqrstuvwxyzABCDEF
+            x-observe-target-package: Host Explorer
+        retry_on_failure:
+            enabled: true
+        sending_queue:
+            num_consumers: 4
+            queue_size: 100
+    prometheusremotewrite/observe:
+        endpoint: https://test.collect.observeinc.com/v1/prometheus
+        headers:
+            authorization: Bearer 12345678901234567890:abcdefghijklmnopqrstuvwxyzABCDEF
+            x-observe-target-package: Host Explorer
+        max_batch_request_parallelism: 5
+        remote_write_queue:
+            num_consumers: 5
+        resource_to_telemetry_conversion:
+            enabled: true
+        send_metadata: true
+extensions:
+    file_storage:
+        directory: C:\ProgramData\Observe\observe-agent\filestorage
+processors:
+    batch:
+        timeout: 5s
+    deltatocumulative: null
+    filter/count:
+        error_mode: ignore
+        metrics:
+            metric:
+                - IsMatch(name, ".*")
+    memory_limiter:
+        check_interval: 1s
+        limit_percentage: 80
+        spike_limit_percentage: 20
+    resourcedetection:
+        detectors:
+            - env
+            - system
+        system:
+            hostname_sources:
+                - os
+            resource_attributes:
+                host.id:
+                    enabled: true
+    resourcedetection/cloud:
+        detectors:
+            - gcp
+            - ecs
+            - ec2
+            - azure
+        override: false
+        timeout: 2s
+    transform/truncate:
+        log_statements:
+            - context: log
+              statements:
+                - truncate_all(attributes, 4095)
+                - truncate_all(resource.attributes, 4095)
+        trace_statements:
+            - context: span
+              statements:
+                - truncate_all(attributes, 4095)
+                - truncate_all(resource.attributes, 4095)
+receivers:
+    nop: null
+service:
+    extensions:
+        - health_check
+        - file_storage
+    pipelines:
+        metrics/count-nop-in:
+            exporters:
+                - count
+            receivers:
+                - nop
+        metrics/count-nop-out:
+            exporters:
+                - nop
+            receivers:
+                - count
diff --git a/packaging/windows/connections/host_monitoring/logs.yaml.tmpl b/packaging/windows/connections/host_monitoring/logs.yaml.tmpl
@@ -14,6 +14,35 @@ receivers:
     retry_on_failure:
       enabled: true
 
+  {{- if .HostMonitoring.Logs.Include }}
+  filelog/host_monitoring:
+    include:
+      {{- range .HostMonitoring.Logs.Include }}
+      - {{ . }}
+      {{- end }}
+    {{- if .HostMonitoring.Logs.Exclude }}
+    exclude:
+      {{- range .HostMonitoring.Logs.Exclude }}
+      - {{ . }}
+      {{- end }}
+    {{- end }}
+    include_file_path: true
+    storage: file_storage
+    retry_on_failure:
+      enabled: true
+    max_log_size: 4MiB
+    operators:
+      - type: filter
+        expr: 'body matches "otel-contrib"'
+      {{- if .HostMonitoring.Logs.AutoMultilineDetection }}
+      - id: multiline-recombine
+        type: recombine
+        combine_field: body
+        # Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
+        is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
+      {{- end }}
+  {{- end }}
+
 service:
   pipelines:
     logs/host_monitoring-windowsevents:
@@ -31,3 +60,21 @@ service:
         {{- end }}
         - batch
       exporters: [otlphttp/observe, count]
+
+    {{- if .HostMonitoring.Logs.Include }}
+    logs/host_monitoring-file:
+      receivers: [filelog/host_monitoring]
+      processors:
+        - memory_limiter
+        - transform/truncate
+        - resourcedetection
+        - resourcedetection/cloud
+        {{- if .HasAttributes }}
+        - attributes/observe_global_attributes
+        {{- end }}
+        {{- if .HasResourceAttributes }}
+        - resource/observe_global_resource_attributes
+        {{- end }}
+        - batch
+      exporters: [otlphttp/observe, count]
+    {{- end }}

Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,7 @@ type snapshotTest struct {`
`37`	`37`	`}`
`38`	`38`
`39`	`39`	`var allSnapshotTests = []snapshotTest{`
	`40`	`+ // Tests with full agent config`
`40`	`41`	`{`
`41`	`42`	`agentConfigPath: "test/snap1-full-agent-config.yaml",`
`42`	`43`	`outputPath: "test/snap1-docker-output.yaml",`
`@@ -52,12 +53,18 @@ var allSnapshotTests = []snapshotTest{`
`52`	`53`	`outputPath: "test/snap1-windows-output.yaml",`
`53`	`54`	`packageType: Windows,`
`54`	`55`	`},`
	`56`	`+ // Tests with minimal agent config`
`55`	`57`	`{`
`56`	`58`	`agentConfigPath: "test/snap2-empty-agent-config.yaml",`
`57`	`59`	`otelConfigPath: "test/snap2-otel-config.yaml",`
`58`	`60`	`outputPath: "test/snap2-with-otel-output.yaml",`
`59`	`61`	`packageType: MacOS,`
`60`	`62`	`},`
	`63`	`+ {`
	`64`	`+ agentConfigPath: "test/snap2-empty-agent-config.yaml",`
	`65`	`+ outputPath: "test/snap2-windows-output.yaml",`
	`66`	`+ packageType: Windows,`
	`67`	`+ },`
`61`	`68`	`}`
`62`	`69`
`63`	`70`	`func Test_RenderOtelConfig(t *testing.T) {`