Skip to content

feat: add default multiline autodetection #215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jun 10, 2025
Merged

feat: add default multiline autodetection #215

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
84cf425
feat: add default multiline autodetection
obs-gh-alexlew Jun 5, 2025
71cf2cc
add to full config
obs-gh-alexlew Jun 5, 2025
22c998d
update snapshot
obs-gh-alexlew Jun 5, 2025
790e3c0
add comment
obs-gh-alexlew Jun 9, 2025
98247fa
merge regexs
obs-gh-alexlew Jun 9, 2025
f558227
update comment
obs-gh-alexlew Jun 9, 2025
4ef1e96
update snapshot
obs-gh-alexlew Jun 9, 2025
72238eb
remove unnecessary escapes
obs-gh-alexlew Jun 10, 2025
f5de34a
Merge branch 'main' into alew/add-multililne-auto
obs-gh-alexlew Jun 10, 2025
ddd33a7
Merge branch 'main' into alew/add-multililne-auto
obs-gh-alexlew Jun 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions integration/scripts/snapshots/docker.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,10 @@ receivers:
operators:
- expr: body matches "otel-contrib"
type: filter
- combine_field: body
id: multiline-recombine
is_first_entry: body matches "^(\\d{4}[-\\/]\\d{2}[-\\/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
type: recombine
retry_on_failure:
enabled: true
storage: file_storage
Expand Down
1 change: 1 addition & 0 deletions integration/scripts/snapshots/full-agent-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ host_monitoring:
enabled: true
exclude: [exclude1, exclude2]
include: [include1, include2]
auto_multiline_detection: true
metrics:
host:
enabled: true
Expand Down
4 changes: 4 additions & 0 deletions integration/scripts/snapshots/linux.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ receivers:
operators:
- expr: body matches "otel-contrib"
type: filter
- combine_field: body
id: multiline-recombine
is_first_entry: body matches "^(\\d{4}[-\\/]\\d{2}[-\\/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
type: recombine
retry_on_failure:
enabled: true
storage: file_storage
Expand Down
29 changes: 17 additions & 12 deletions internal/commands/initconfig/initconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,19 @@ import (
)

var (
config_path string
token string
observe_url string
cloud_resource_detectors []string
resource_attributes map[string]string
forwarding_metrics_format string
self_monitoring_enabled bool
host_monitoring_enabled bool
host_monitoring_logs_enabled bool
host_monitoring_logs_include []string
host_monitoring_metrics_host_enabled bool
host_monitoring_metrics_process_enabled bool
config_path string
token string
observe_url string
cloud_resource_detectors []string
resource_attributes map[string]string
forwarding_metrics_format string
self_monitoring_enabled bool
host_monitoring_enabled bool
host_monitoring_logs_enabled bool
host_monitoring_logs_include []string
host_monitoring_logs_auto_multiline_detection bool
host_monitoring_metrics_host_enabled bool
host_monitoring_metrics_process_enabled bool
)

func NewConfigureCmd(v *viper.Viper) *cobra.Command {
Expand Down Expand Up @@ -106,6 +107,10 @@ func RegisterConfigFlags(cmd *cobra.Command, v *viper.Viper) {
cmd.PersistentFlags().StringSliceVar(&host_monitoring_logs_include, "host_monitoring::logs::include", nil, "Set host monitoring log include paths")
v.BindPFlag("host_monitoring::logs::include", cmd.PersistentFlags().Lookup("host_monitoring::logs::include"))

cmd.PersistentFlags().BoolVar(&host_monitoring_logs_auto_multiline_detection, "host_monitoring::logs::auto_multiline_detection", false, "Enable host monitoring log auto multiline detection")
v.BindPFlag("host_monitoring::logs::auto_multiline_detection", cmd.PersistentFlags().Lookup("host_monitoring::logs::auto_multiline_detection"))
v.SetDefault("host_monitoring::logs::auto_multiline_detection", false)

cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_host_enabled, "host_monitoring::metrics::host::enabled", true, "Enable host monitoring host metrics")
v.BindPFlag("host_monitoring::metrics::host::enabled", cmd.PersistentFlags().Lookup("host_monitoring::metrics::host::enabled"))
v.SetDefault("host_monitoring::metrics::host::enabled", true)
Expand Down
7 changes: 4 additions & 3 deletions internal/config/configschema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ import (
)

type HostMonitoringLogsConfig struct {
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
Include []string `yaml:"include,omitempty" mapstructure:"include"`
Exclude []string `yaml:"exclude,omitempty" mapstructure:"exclude"`
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
Include []string `yaml:"include,omitempty" mapstructure:"include"`
Exclude []string `yaml:"exclude,omitempty" mapstructure:"exclude"`
AutoMultilineDetection bool `yaml:"auto_multiline_detection" mapstructure:"auto_multiline_detection"`
}

type HostMonitoringHostMetricsConfig struct {
Expand Down
7 changes: 7 additions & 0 deletions packaging/docker/observe-agent/connections/host_monitoring/logs.yaml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ receivers:
operators:
- type: filter
expr: 'body matches "otel-contrib"'
{{- if .HostMonitoring.Logs.AutoMultilineDetection }}
- id: multiline-recombine
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this operator differ from the top level multiline config option?
https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.124.0/receiver/filelogreceiver#example---multiline-logs-parsing
If they work the same, I think it's better to use the multiline option since it can be overridden by users more easily than a field in an array.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so i think specifically the order of operations between the multiline top level config and then the operators pipelines means that we need to define this as another operator after the container operator for kubernetes specifically. I dont actually know if its necessary in this case but figured it might be best to do it the same way everywhere so that the behavior is most likely to be similar.

type: recombine
combine_field: body
# Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
is_first_entry: body matches "^(\\d{4}[-\\/]\\d{2}[-\\/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
{{- end }}

journald/host_monitoring:
units:
Expand Down
7 changes: 7 additions & 0 deletions packaging/linux/connections/host_monitoring/logs.yaml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ receivers:
operators:
- type: filter
expr: 'body matches "otel-contrib"'
{{- if .HostMonitoring.Logs.AutoMultilineDetection }}
- id: multiline-recombine
type: recombine
combine_field: body
# Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
is_first_entry: body matches "^(\\d{4}[-\\/]\\d{2}[-\\/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
{{- end }}

journald/host_monitoring:
units:
Expand Down
7 changes: 7 additions & 0 deletions packaging/macos/connections/host_monitoring/logs.yaml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ receivers:
operators:
- type: filter
expr: 'body matches "otel-contrib"'
{{- if .HostMonitoring.Logs.AutoMultilineDetection }}
- id: multiline-recombine
type: recombine
combine_field: body
# Regex is just 3 different pattern's OR'd together to match the 4 timestamp formats `2021-03-28 13:45:30`, `2023-03-28T14:33:53.743350Z`, `Jun 14 15:16:01`, `2024/05/16 19:46:15`
is_first_entry: body matches "^(\\d{4}[-\\/]\\d{2}[-\\/]\\d{2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z?|[A-Za-z]+ \\d{1,2} \\d{2}:\\d{2}:\\d{2})"
{{- end }}

service:
pipelines:
Expand Down
Loading