[proofpoint_tap] Clean up null handling (#9153)

- Combine 'is null or not contains' checks.
- Combine 'not null and is/not value' checks.
- Remove redundant null-safe operator.

Author: chrisberkhout

packages/proofpoint_tap/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.3"
+  changes:
+    - description: Clean up null handling
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9153
 - version: "1.16.2"
   changes:
     - description: Add error.message ECS field mapping.

packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml

@@ -182,7 +182,7 @@ processors:
       ignore_failure: true
   - remove:
       field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
+      if: ctx.tags?.contains('preserve_original_event') != true
       ignore_failure: true
       ignore_missing: true
   - remove:

packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml

@@ -182,7 +182,7 @@ processors:
       ignore_failure: true
   - remove:
       field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
+      if: ctx.tags?.contains('preserve_original_event') != true
       ignore_failure: true
       ignore_missing: true
   - remove:

packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml

@@ -99,7 +99,7 @@ processors:
           allow_duplicates: false
           ignore_failure: true
       ignore_failure: true
-      if: ctx.json?.messageParts != null && ctx.json.messageParts instanceof List
+      if: ctx.json?.messageParts instanceof List
   - foreach:
       field: json.messageParts
       processor:
@@ -109,7 +109,7 @@ processors:
           allow_duplicates: false
           ignore_failure: true
       ignore_failure: true
-      if: ctx.json?.messageParts != null && ctx.json.messageParts instanceof List
+      if: ctx.json?.messageParts instanceof List
   - rename:
       field: json.ccAddresses
       target_field: email.cc.address
@@ -168,7 +168,7 @@ processors:
           allow_duplicates: false
           ignore_failure: true
       ignore_failure: true
-      if: ctx.json?.recipient != null && ctx.json.recipient instanceof List
+      if: ctx.json?.recipient instanceof List
   - rename:
       field: json.xmailer
       target_field: email.x_mailer
@@ -191,7 +191,7 @@ processors:
             - _ingest._value.sandboxStatus
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -200,7 +200,7 @@ processors:
           target_field: _ingest._value.file.mime_type
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -209,7 +209,7 @@ processors:
           target_field: _ingest._value.file.hash.md5
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -218,7 +218,7 @@ processors:
           target_field: _ingest._value.file.hash.sha256
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -227,14 +227,14 @@ processors:
           target_field: _ingest._value.file.name
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - script:
       description: Adding hash in related.hash from artifact field.
       lang: painless
       ignore_failure: true
       source: |
         if (ctx.json?.threatsInfoMap instanceof List) {
-            for (artifact in ctx.json?.threatsInfoMap) {
+            for (artifact in ctx.json.threatsInfoMap) {
                 def flag = true;
                 def str = artifact.threat.toLowerCase();
                 if (str?.length() == 64) {
@@ -331,7 +331,7 @@ processors:
             - _ingest._value.sha256
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.message_parts != null && ctx.proofpoint_tap.message_blocked.message_parts instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.message_parts instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.message_parts
       processor:
@@ -340,7 +340,7 @@ processors:
           target_field: _ingest._value.o_content_type
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.message_parts != null && ctx.proofpoint_tap.message_blocked.message_parts instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.message_parts instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.message_parts
       processor:
@@ -349,7 +349,7 @@ processors:
           target_field: _ingest._value.sandbox_status
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.message_parts != null && ctx.proofpoint_tap.message_blocked.message_parts instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.message_parts instanceof List
   - convert:
       field: json.messageSize
       target_field: proofpoint_tap.message_blocked.message_size
@@ -415,7 +415,7 @@ processors:
           target_field: _ingest._value.campaign_id
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
@@ -424,7 +424,7 @@ processors:
           target_field: _ingest._value.threat.artifact
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
@@ -433,7 +433,7 @@ processors:
           target_field: _ingest._value.threat.id
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
@@ -442,7 +442,7 @@ processors:
           target_field: _ingest._value.threat.status
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
@@ -453,15 +453,15 @@ processors:
           formats:
             - ISO8601
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
         remove:
           field: _ingest._value.threatTime
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
@@ -470,7 +470,7 @@ processors:
           target_field: _ingest._value.threat.type
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_blocked.threat_info_map
       processor:
@@ -479,10 +479,10 @@ processors:
           target_field: _ingest._value.threat.url
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map != null && ctx.proofpoint_tap.message_blocked.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List
   - remove:
       field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
+      if: ctx.tags?.contains('preserve_original_event') != true
       ignore_failure: true
       ignore_missing: true
   - remove:

packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml

@@ -95,7 +95,7 @@ processors:
           allow_duplicates: false
           ignore_failure: true
       ignore_failure: true
-      if: ctx.json?.messageParts != null && ctx.json.messageParts instanceof List
+      if: ctx.json?.messageParts instanceof List
   - foreach:
       field: json.messageParts
       processor:
@@ -105,7 +105,7 @@ processors:
           allow_duplicates: false
           ignore_failure: true
       ignore_failure: true
-      if: ctx.json?.messageParts != null && ctx.json.messageParts instanceof List
+      if: ctx.json?.messageParts instanceof List
   - rename:
       field: json.ccAddresses
       target_field: email.cc.address
@@ -164,7 +164,7 @@ processors:
           allow_duplicates: false
           ignore_failure: true
       ignore_failure: true
-      if: ctx.json?.recipient != null && ctx.json.recipient instanceof List
+      if: ctx.json?.recipient instanceof List
   - rename:
       field: json.xmailer
       target_field: email.x_mailer
@@ -187,7 +187,7 @@ processors:
             - _ingest._value.sandboxStatus
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -196,7 +196,7 @@ processors:
           target_field: _ingest._value.file.mime_type
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -205,7 +205,7 @@ processors:
           target_field: _ingest._value.file.hash.md5
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -214,7 +214,7 @@ processors:
           target_field: _ingest._value.file.hash.sha256
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - foreach:
       field: email.attachments
       processor:
@@ -223,14 +223,14 @@ processors:
           target_field: _ingest._value.file.name
           ignore_missing: true
       ignore_failure: true
-      if: ctx.email?.attachments != null && ctx.email.attachments instanceof List
+      if: ctx.email?.attachments instanceof List
   - script:
       description: Adding hash in related.hash from artifact field.
       lang: painless
       ignore_failure: true
       source: |
         if (ctx.json?.threatsInfoMap instanceof List) {
-            for (artifact in ctx.json?.threatsInfoMap) {
+            for (artifact in ctx.json.threatsInfoMap) {
                 def flag = true;
                 def str = artifact.threat.toLowerCase();
                 if (str?.length() == 64) {
@@ -319,7 +319,7 @@ processors:
             - _ingest._value.sha256
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.message_parts != null && ctx.proofpoint_tap.message_delivered.message_parts instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.message_parts instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.message_parts
       processor:
@@ -328,7 +328,7 @@ processors:
           target_field: _ingest._value.o_content_type
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.message_parts != null && ctx.proofpoint_tap.message_delivered.message_parts instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.message_parts instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.message_parts
       processor:
@@ -337,7 +337,7 @@ processors:
           target_field: _ingest._value.sandbox_status
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.message_parts != null && ctx.proofpoint_tap.message_delivered.message_parts instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.message_parts instanceof List
   - convert:
       field: json.messageSize
       target_field: proofpoint_tap.message_delivered.message_size
@@ -403,7 +403,7 @@ processors:
           target_field: _ingest._value.campaign_id
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
@@ -412,7 +412,7 @@ processors:
           target_field: _ingest._value.threat.artifact
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
@@ -421,7 +421,7 @@ processors:
           target_field: _ingest._value.threat.id
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
@@ -430,7 +430,7 @@ processors:
           target_field: _ingest._value.threat.status
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
@@ -441,15 +441,15 @@ processors:
           formats:
             - ISO8601
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
         remove:
           field: _ingest._value.threatTime
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
@@ -458,7 +458,7 @@ processors:
           target_field: _ingest._value.threat.type
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - foreach:
       field: proofpoint_tap.message_delivered.threat_info_map
       processor:
@@ -467,10 +467,10 @@ processors:
           target_field: _ingest._value.threat.url
           ignore_missing: true
       ignore_failure: true
-      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map != null && ctx.proofpoint_tap.message_delivered.threat_info_map instanceof List
+      if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List
   - remove:
       field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
+      if: ctx.tags?.contains('preserve_original_event') != true
       ignore_failure: true
       ignore_missing: true
   - remove: