Skip to content

Commit 1a1a7d9

Browse files
SimonKoettingSimonKoetting
committed
Merge branch 'zscaler_zpa_remove_empty_json_values' of github.com:SimonKoetting/integrations into zscaler_zpa_remove_empty_json_values
2 parents e90b963 + fa77642 commit 1a1a7d9

File tree

11 files changed

+69
-47
lines changed

11 files changed

+69
-47
lines changed

‎packages/nginx/changelog.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "2.2.0"
3+
changes:
4+
- description: Added optional parsing for request duration in access-log.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/14126
27
- version: "2.1.0"
38
changes:
49
- description: Allow @custom pipeline access to event.original without setting preserve_original_event.

‎packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36"
1010
127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
1111
127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
12-
127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
12+
127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 45.324 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
1313
lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36"
1414
lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] "GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36"

‎packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"web"
1212
],
1313
"created": "2020-04-28T11:07:58.223Z",
14-
"ingested": "2024-06-18T05:50:06.491934242Z",
14+
"ingested": "2025-06-03T12:31:45.476662367Z",
1515
"kind": "event",
1616
"original": "67.43.156.13 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"",
1717
"outcome": "success",
@@ -91,7 +91,7 @@
9191
"web"
9292
],
9393
"created": "2020-04-28T11:07:58.223Z",
94-
"ingested": "2024-06-18T05:50:06.491955973Z",
94+
"ingested": "2025-06-03T12:31:45.476686597Z",
9595
"kind": "event",
9696
"original": "67.43.156.13 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"",
9797
"outcome": "failure",
@@ -173,7 +173,7 @@
173173
"web"
174174
],
175175
"created": "2020-04-28T11:07:58.223Z",
176-
"ingested": "2024-06-18T05:50:06.491959697Z",
176+
"ingested": "2025-06-03T12:31:45.476689457Z",
177177
"kind": "event",
178178
"original": "67.43.156.13 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"",
179179
"outcome": "failure",
@@ -253,7 +253,7 @@
253253
"web"
254254
],
255255
"created": "2020-04-28T11:07:58.223Z",
256-
"ingested": "2024-06-18T05:50:06.491962764Z",
256+
"ingested": "2025-06-03T12:31:45.476691307Z",
257257
"kind": "event",
258258
"original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
259259
"outcome": "success",
@@ -333,7 +333,7 @@
333333
"web"
334334
],
335335
"created": "2020-04-28T11:07:58.223Z",
336-
"ingested": "2024-06-18T05:50:06.491965569Z",
336+
"ingested": "2025-06-03T12:31:45.476693107Z",
337337
"kind": "event",
338338
"original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
339339
"outcome": "failure",
@@ -415,7 +415,7 @@
415415
"web"
416416
],
417417
"created": "2020-04-28T11:07:58.223Z",
418-
"ingested": "2024-06-18T05:50:06.491968355Z",
418+
"ingested": "2025-06-03T12:31:45.476694737Z",
419419
"kind": "event",
420420
"original": "67.43.156.13 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
421421
"outcome": "failure",
@@ -495,7 +495,7 @@
495495
"web"
496496
],
497497
"created": "2020-04-28T11:07:58.223Z",
498-
"ingested": "2024-06-18T05:50:06.491971121Z",
498+
"ingested": "2025-06-03T12:31:45.476696247Z",
499499
"kind": "event",
500500
"original": "67.43.156.13 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
501501
"outcome": "failure",
@@ -575,7 +575,7 @@
575575
"web"
576576
],
577577
"created": "2020-04-28T11:07:58.223Z",
578-
"ingested": "2024-06-18T05:50:06.491973900Z",
578+
"ingested": "2025-06-03T12:31:45.476697827Z",
579579
"kind": "event",
580580
"original": "67.43.156.13 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
581581
"outcome": "failure",
@@ -655,7 +655,7 @@
655655
"web"
656656
],
657657
"created": "2020-04-28T11:07:58.223Z",
658-
"ingested": "2024-06-18T05:50:06.491976741Z",
658+
"ingested": "2025-06-03T12:31:45.476699327Z",
659659
"kind": "event",
660660
"original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
661661
"outcome": "failure",
@@ -723,7 +723,7 @@
723723
"web"
724724
],
725725
"created": "2020-04-28T11:07:58.223Z",
726-
"ingested": "2024-06-18T05:50:06.491979537Z",
726+
"ingested": "2025-06-03T12:31:45.476700857Z",
727727
"kind": "event",
728728
"original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"",
729729
"outcome": "success",
@@ -791,7 +791,7 @@
791791
"web"
792792
],
793793
"created": "2020-04-28T11:07:58.223Z",
794-
"ingested": "2024-06-18T05:50:06.491982307Z",
794+
"ingested": "2025-06-03T12:31:45.476702357Z",
795795
"kind": "event",
796796
"original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"",
797797
"outcome": "success",
@@ -859,9 +859,9 @@
859859
"web"
860860
],
861861
"created": "2020-04-28T11:07:58.223Z",
862-
"ingested": "2024-06-18T05:50:06.491985407Z",
862+
"ingested": "2025-06-03T12:31:45.476703857Z",
863863
"kind": "event",
864-
"original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"",
864+
"original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 45.324 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"",
865865
"outcome": "failure",
866866
"type": [
867867
"access"
@@ -883,7 +883,8 @@
883883
"access": {
884884
"remote_ip_list": [
885885
"127.0.0.1"
886-
]
886+
],
887+
"response_time": 45324
887888
}
888889
},
889890
"related": {

‎packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"web"
1212
],
1313
"created": "2020-04-28T11:07:58.223Z",
14-
"ingested": "2024-06-18T05:50:06.958062590Z",
14+
"ingested": "2025-06-03T12:31:49.396736609Z",
1515
"kind": "event",
1616
"original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"",
1717
"outcome": "success",
@@ -81,7 +81,7 @@
8181
"web"
8282
],
8383
"created": "2020-04-28T11:07:58.223Z",
84-
"ingested": "2024-06-18T05:50:06.958087114Z",
84+
"ingested": "2025-06-03T12:31:49.396750059Z",
8585
"kind": "event",
8686
"original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
8787
"outcome": "failure",
@@ -149,7 +149,7 @@
149149
"web"
150150
],
151151
"created": "2020-04-28T11:07:58.223Z",
152-
"ingested": "2024-06-18T05:50:06.958091109Z",
152+
"ingested": "2025-06-03T12:31:49.396752609Z",
153153
"kind": "event",
154154
"original": "10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"",
155155
"outcome": "success",
@@ -231,7 +231,7 @@
231231
"web"
232232
],
233233
"created": "2020-04-28T11:07:58.223Z",
234-
"ingested": "2024-06-18T05:50:06.958094069Z",
234+
"ingested": "2025-06-03T12:31:49.396754329Z",
235235
"kind": "event",
236236
"original": "67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\n\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"\n2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"",
237237
"outcome": "success",
@@ -310,7 +310,7 @@
310310
"web"
311311
],
312312
"created": "2020-04-28T11:07:58.223Z",
313-
"ingested": "2024-06-18T05:50:06.958100802Z",
313+
"ingested": "2025-06-03T12:31:49.396756009Z",
314314
"kind": "event",
315315
"original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nunix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"\nlocalhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nlocalhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\n",
316316
"outcome": "failure",

‎packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ example.com:80 "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/20
77
67.43.156.15:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-"
88
example.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] "hello" 400 173 "-" "-"
99
67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
10-
example.com localhost, localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
10+
example.com localhost, localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 25.647 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"

‎packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"web"
1515
],
1616
"created": "2020-04-28T11:07:58.223Z",
17-
"ingested": "2024-06-18T05:50:07.175960670Z",
17+
"ingested": "2025-06-03T12:31:52.934925962Z",
1818
"kind": "event",
1919
"original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com 10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com:80 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\nexample.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"",
2020
"outcome": "success",
@@ -88,7 +88,7 @@
8888
"web"
8989
],
9090
"created": "2020-04-28T11:07:58.223Z",
91-
"ingested": "2024-06-18T05:50:07.175984368Z",
91+
"ingested": "2025-06-03T12:31:52.934939492Z",
9292
"kind": "event",
9393
"original": "67.43.156.15 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"",
9494
"outcome": "failure",
@@ -167,7 +167,7 @@
167167
"web"
168168
],
169169
"created": "2020-04-28T11:07:58.223Z",
170-
"ingested": "2024-06-18T05:50:07.175988030Z",
170+
"ingested": "2025-06-03T12:31:52.934943871Z",
171171
"kind": "event",
172172
"original": "67.43.156.15:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nexample.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"",
173173
"outcome": "failure",
@@ -218,9 +218,9 @@
218218
"web"
219219
],
220220
"created": "2020-04-28T11:07:58.223Z",
221-
"ingested": "2024-06-18T05:50:07.175991090Z",
221+
"ingested": "2025-06-03T12:31:52.934945341Z",
222222
"kind": "event",
223-
"original": "67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
223+
"original": "67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 25.647 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
224224
"outcome": "success",
225225
"type": [
226226
"access"

‎packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ processors:
2121
patterns:
2222
- (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address})
2323
- (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}"
24-
%{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}
24+
%{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}(?:\s%{NUMBER:nginx.access.response_time})?
2525
"(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})"
2626
pattern_definitions:
2727
NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?
@@ -189,6 +189,17 @@ processors:
189189
field: related.user
190190
value: "{{user.name}}"
191191
if: "ctx?.user?.name != null"
192+
- convert:
193+
field: nginx.access.response_time
194+
tag: convert_response_time_to_double
195+
type: double
196+
ignore_missing: true
197+
- script:
198+
lang: painless
199+
description: This script multiplies the second value of response_time * 1000 to convert to milliseconds.
200+
source: |
201+
ctx.nginx.access.response_time = (long) (ctx.nginx.access.response_time * 1000)
202+
if: ctx.nginx?.access?.response_time != null
192203
- script:
193204
lang: painless
194205
description: This script processor iterates over the whole document to remove fields with null values.

0 commit comments

Comments
 (0)