[ti_opencti] Fix processing of externalReferences

packages/ti_opencti/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.3.2"
+  changes:
+    - description: Fix processing of externalReferences.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8556
 - version: "0.3.1"
   changes:
     - description: Fix event.original field type conflict

packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json

@@ -0,0 +1,77 @@
+{
+  "events": [
+    {
+      "confidence": 0,
+      "created": "2023-11-02T00:17:00.295Z",
+      "createdBy": {
+        "identity_class": "organization",
+        "name": "Stopforumspam"
+      },
+      "description": "Stopforumspam",
+      "externalReferences": {
+        "edges": [
+          {
+            "node": {
+              "description": "Stopforumspam feed URL",
+              "source_name": "stopforumspam",
+              "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole_filtered_50000.txt"
+            }
+          },
+          {
+            "node": {
+              "external_id": null,
+              "source_name": "MISC",
+              "url": "https://example.com/CVE-0079-1234",
+              "description": null
+            }
+          }
+        ]
+      },
+      "id": "fcfa872e-a8b6-4525-847e-f3c756b70035",
+      "is_inferred": false,
+      "killChainPhases": {
+        "edges": null
+      },
+      "lang": "en",
+      "modified": "2023-11-09T23:22:20.586Z",
+      "name": "freelifetimexxxdates.com",
+      "objectLabel": {
+        "edges": [
+          {
+            "node": {
+              "value": "spam"
+            }
+          }
+        ]
+      },
+      "objectMarking": {
+        "edges": null
+      },
+      "observables": {
+        "edges": [
+          {
+            "node": {
+              "entity_type": "Domain-Name",
+              "id": "cc34949a-5a6f-4595-afec-c3bf98c62a7d",
+              "observable_value": "freelifetimexxxdates.com",
+              "standard_id": "domain-name--726e8863-8941-5a1b-b345-1f0131902233",
+              "value": "freelifetimexxxdates.com"
+            }
+          }
+        ],
+        "pageInfo": {
+          "globalCount": 1
+        }
+      },
+      "pattern": "[domain-name:value = 'freelifetimexxxdates.com']",
+      "pattern_type": "stix",
+      "revoked": false,
+      "standard_id": "indicator--08a7e875-2ce4-50ab-a8de-2915addd93c4",
+      "valid_from": "2023-11-09T23:22:19.426Z",
+      "valid_until": "2024-11-08T23:22:19.426Z",
+      "x_opencti_detection": false,
+      "x_opencti_main_observable_type": "Domain-Name",
+      "x_opencti_score": 60
+    }
+  ]
+}

packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json-expected.json

@@ -0,0 +1,88 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "threat"
+                ],
+                "created": "2023-11-02T00:17:00.295Z",
+                "id": "fcfa872e-a8b6-4525-847e-f3c756b70035",
+                "kind": "enrichment",
+                "type": [
+                    "indicator"
+                ]
+            },
+            "opencti": {
+                "indicator": {
+                    "creator_identity_class": "organization",
+                    "detection": false,
+                    "external_reference": {
+                        "description": "Stopforumspam feed URL",
+                        "source_name": [
+                            "stopforumspam",
+                            "MISC"
+                        ],
+                        "url": [
+                            "https://www.stopforumspam.com/downloads/toxic_domains_whole_filtered_50000.txt",
+                            "https://example.com/CVE-0079-1234"
+                        ]
+                    },
+                    "is_inferred": false,
+                    "lang": "en",
+                    "observables_count": 1,
+                    "pattern": "[domain-name:value = 'freelifetimexxxdates.com']",
+                    "pattern_type": "stix",
+                    "revoked": false,
+                    "score": 60,
+                    "standard_id": "indicator--08a7e875-2ce4-50ab-a8de-2915addd93c4",
+                    "valid_from": "2023-11-09T23:22:19.426Z",
+                    "valid_until": "2024-11-08T23:22:19.426Z"
+                },
+                "observable": {
+                    "domain_name": {
+                        "entity_type": "Domain-Name",
+                        "id": "cc34949a-5a6f-4595-afec-c3bf98c62a7d",
+                        "standard_id": "domain-name--726e8863-8941-5a1b-b345-1f0131902233",
+                        "value": "freelifetimexxxdates.com"
+                    }
+                }
+            },
+            "related": {
+                "hosts": [
+                    "freelifetimexxxdates.com"
+                ]
+            },
+            "tags": [
+                "forwarded",
+                "opencti-indicator",
+                "spam",
+                "ecs-indicator-detail"
+            ],
+            "threat": {
+                "feed": {
+                    "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd",
+                    "description": "Indicator data from OpenCTI",
+                    "name": "OpenCTI",
+                    "reference": "https://docs.opencti.io/latest/usage/overview/"
+                },
+                "indicator": {
+                    "confidence": "None",
+                    "description": "Stopforumspam",
+                    "modified_at": "2023-11-09T23:22:20.586Z",
+                    "name": "freelifetimexxxdates.com",
+                    "provider": "Stopforumspam",
+                    "reference": "https://demo.opencti.io/dashboard/observations/indicators/fcfa872e-a8b6-4525-847e-f3c756b70035",
+                    "type": "domain-name",
+                    "url": {
+                        "domain": "freelifetimexxxdates.com",
+                        "registered_domain": "freelifetimexxxdates.com",
+                        "top_level_domain": "com"
+                    }
+                }
+            }
+        }
+    ]
+}

packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml

@@ -146,7 +146,10 @@ processors:
       ignore_missing: true
 
   - gsub:
-      if: '!ctx.objectMarking.edges.isEmpty() && ctx.objectMarking.edges[0].node.definition_type == "TLP"'
+      if: |
+        ctx.objectMarking.edges != null &&
+        !ctx.objectMarking.edges.isEmpty() &&
+        ctx.objectMarking.edges[0].node.definition_type == "TLP"
       field: objectMarking.edges.0.node.definition
       pattern: '^TLP:'
       replacement: ''
@@ -156,6 +159,7 @@ processors:
 
   - foreach:
       field: objectLabel.edges
+      ignore_missing: true
       processor:
         append:
           field: tags
@@ -166,19 +170,42 @@ processors:
 
   - foreach:
       field: killChainPhases.edges
+      ignore_missing: true
       processor:
         append:
           field: opencti.indicator.kill_chain_phase
           value: "[{{{_ingest._value.node.kill_chain_name}}}] {{{_ingest._value.node.phase_name}}}"
   - remove:
       field: killChainPhases
 
-  - foreach:
-      field: externalReferences.edges
-      processor:
-        append:
-          field: opencti.indicator.external_reference
-          value: "{{{_ingest._value.node}}}"
+  - script:
+      description: Move external reference nodes to their target location, without nulls
+      lang: painless
+      source: |
+        ArrayList edges = ctx.externalReferences.edges;
+        if (edges == null) {
+          return;
+        }
+        for (int i = 0; i < edges.length; i++) {
+          if (!ctx.opencti?.indicator?.containsKey('external_reference') == true) {
+            if (!ctx.containsKey('opencti')) {
+              ctx.opencti = [:];
+            }
+            if (!ctx.opencti.containsKey('indicator')) {
+              ctx.opencti.indicator = [:];
+            }
+            if (!ctx.opencti.indicator.containsKey('external_reference')) {
+              ctx.opencti.indicator.external_reference = [];
+            }
+          }
+          def newNode = [:];
+          for (def key : edges[i]['node'].keySet()) {
+            if (edges[i]['node'][key] != null) {
+              newNode[key] = edges[i]['node'][key];
+            }
+          }
+          ctx.opencti.indicator.external_reference.add(newNode);
+        }
   - remove:
       field: externalReferences
 
@@ -389,7 +416,7 @@ processors:
             return obj;
           }
         }
-        if (ctx.opencti?.containsKey('observable')) {
+        if (ctx.opencti?.containsKey('observable') == true) {
           ctx.opencti.observable = dropNulls(ctx.opencti.observable);
           for (def key : ctx.opencti.observable.keySet()) {
             if (ctx.opencti.observable[key].size() == 0) {
@@ -754,6 +781,7 @@ processors:
   ##############################################
   # Restructure to avoid non-leaf lists, under #
   # - opencti.observable.*                     #
+  # - opencti.indicator.external_reference     #
   # - threat.indicator.as                      #
   # - threat.indicator.file                    #
   # - threat.indicator.registry                #
@@ -795,11 +823,14 @@ processors:
           }
           return merged;
         }
-        if (ctx.opencti?.containsKey('observable')) {
+        if (ctx.opencti?.containsKey('observable') == true) {
           for (def key : ctx.opencti.observable.keySet()) {
             ctx.opencti.observable[key] = mergeListOfMaps(ctx.opencti.observable[key]);
           }
         }
+        if (ctx.opencti?.indicator?.containsKey('external_reference') == true) {
+          ctx.opencti.indicator.external_reference = mergeListOfMaps(ctx.opencti.indicator.external_reference);
+        }
         if (ctx.threat.indicator.containsKey('file')) {
           ctx.threat.indicator.file = mergeListOfMaps(ctx.threat.indicator.file);
         }

packages/ti_opencti/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: ti_opencti
 title: OpenCTI
-version: "0.3.1"
+version: "0.3.2"
 description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent."
 type: integration
 source: