Skip to content

Commit 8a4fbd0

Browse files
feat: add default multiline autodetection (#215)
### Description OB-XXX Please explain the changes you made here. ### Checklist - [ ] Created tests which fail without the change (if possible) - [ ] Extended the README / documentation, if necessary
1 parent b364e4a commit 8a4fbd0

File tree

8 files changed

+51
-15
lines changed

8 files changed

+51
-15
lines changed

integration/scripts/snapshots/docker.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,10 @@ receivers:
151151
operators:
152152
- expr: body matches "otel-contrib"
153153
type: filter
154+
- combine_field: body
155+
id: multiline-recombine
156+
is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
157+
type: recombine
154158
retry_on_failure:
155159
enabled: true
156160
storage: file_storage

integration/scripts/snapshots/full-agent-config.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ host_monitoring:
4343
enabled: true
4444
exclude: [exclude1, exclude2]
4545
include: [include1, include2]
46+
auto_multiline_detection: true
4647
metrics:
4748
host:
4849
enabled: true

integration/scripts/snapshots/linux.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ receivers:
148148
operators:
149149
- expr: body matches "otel-contrib"
150150
type: filter
151+
- combine_field: body
152+
id: multiline-recombine
153+
is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
154+
type: recombine
151155
retry_on_failure:
152156
enabled: true
153157
storage: file_storage

internal/commands/initconfig/initconfig.go

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,19 @@ import (
1414
)
1515

1616
var (
17-
config_path string
18-
token string
19-
observe_url string
20-
cloud_resource_detectors []string
21-
resource_attributes map[string]string
22-
forwarding_metrics_format string
23-
self_monitoring_enabled bool
24-
host_monitoring_enabled bool
25-
host_monitoring_logs_enabled bool
26-
host_monitoring_logs_include []string
27-
host_monitoring_metrics_host_enabled bool
28-
host_monitoring_metrics_process_enabled bool
17+
config_path string
18+
token string
19+
observe_url string
20+
cloud_resource_detectors []string
21+
resource_attributes map[string]string
22+
forwarding_metrics_format string
23+
self_monitoring_enabled bool
24+
host_monitoring_enabled bool
25+
host_monitoring_logs_enabled bool
26+
host_monitoring_logs_include []string
27+
host_monitoring_logs_auto_multiline_detection bool
28+
host_monitoring_metrics_host_enabled bool
29+
host_monitoring_metrics_process_enabled bool
2930
)
3031

3132
func NewConfigureCmd(v *viper.Viper) *cobra.Command {
@@ -106,6 +107,10 @@ func RegisterConfigFlags(cmd *cobra.Command, v *viper.Viper) {
106107
cmd.PersistentFlags().StringSliceVar(&host_monitoring_logs_include, "host_monitoring::logs::include", nil, "Set host monitoring log include paths")
107108
v.BindPFlag("host_monitoring::logs::include", cmd.PersistentFlags().Lookup("host_monitoring::logs::include"))
108109

110+
cmd.PersistentFlags().BoolVar(&host_monitoring_logs_auto_multiline_detection, "host_monitoring::logs::auto_multiline_detection", false, "Enable host monitoring log auto multiline detection")
111+
v.BindPFlag("host_monitoring::logs::auto_multiline_detection", cmd.PersistentFlags().Lookup("host_monitoring::logs::auto_multiline_detection"))
112+
v.SetDefault("host_monitoring::logs::auto_multiline_detection", false)
113+
109114
cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_host_enabled, "host_monitoring::metrics::host::enabled", true, "Enable host monitoring host metrics")
110115
v.BindPFlag("host_monitoring::metrics::host::enabled", cmd.PersistentFlags().Lookup("host_monitoring::metrics::host::enabled"))
111116
v.SetDefault("host_monitoring::metrics::host::enabled", true)

internal/config/configschema.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@ import (
1212
)
1313

1414
type HostMonitoringLogsConfig struct {
15-
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
16-
Include []string `yaml:"include,omitempty" mapstructure:"include"`
17-
Exclude []string `yaml:"exclude,omitempty" mapstructure:"exclude"`
15+
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
16+
Include []string `yaml:"include,omitempty" mapstructure:"include"`
17+
Exclude []string `yaml:"exclude,omitempty" mapstructure:"exclude"`
18+
AutoMultilineDetection bool `yaml:"auto_multiline_detection" mapstructure:"auto_multiline_detection"`
1819
}
1920

2021
type HostMonitoringHostMetricsConfig struct {

packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,13 @@ receivers:
2323
operators:
2424
- type: filter
2525
expr: 'body matches "otel-contrib"'
26+
{{- if .HostMonitoring.Logs.AutoMultilineDetection }}
27+
- id: multiline-recombine
28+
type: recombine
29+
combine_field: body
30+
# Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
31+
is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
32+
{{- end }}
2633

2734
journald/host_monitoring:
2835
units:

packaging/linux/connections/host_monitoring/logs.yaml.tmpl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,13 @@ receivers:
2323
operators:
2424
- type: filter
2525
expr: 'body matches "otel-contrib"'
26+
{{- if .HostMonitoring.Logs.AutoMultilineDetection }}
27+
- id: multiline-recombine
28+
type: recombine
29+
combine_field: body
30+
# Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
31+
is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
32+
{{- end }}
2633

2734
journald/host_monitoring:
2835
units:

packaging/macos/connections/host_monitoring/logs.yaml.tmpl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ receivers:
2222
operators:
2323
- type: filter
2424
expr: 'body matches "otel-contrib"'
25+
{{- if .HostMonitoring.Logs.AutoMultilineDetection }}
26+
- id: multiline-recombine
27+
type: recombine
28+
combine_field: body
29+
# Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
30+
is_first_entry: body matches "^(\\d{4}[-/]\\d{2}[-/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
31+
{{- end }}
2532

2633
service:
2734
pipelines:

0 commit comments

Comments
 (0)