diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 78d4824951b..a474edb61e9 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.2" + changes: + - description: Fix processing of externalReferences. + type: bugfix + link: https://github.com/elastic/integrations/pull/8556 - version: "0.3.1" changes: - description: Fix event.original field type conflict diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json new file mode 100644 index 00000000000..84a9922cb08 --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json @@ -0,0 +1,77 @@ +{ + "events": [ + { + "confidence": 0, + "created": "2023-11-02T00:17:00.295Z", + "createdBy": { + "identity_class": "organization", + "name": "Stopforumspam" + }, + "description": "Stopforumspam", + "externalReferences": { + "edges": [ + { + "node": { + "description": "Stopforumspam feed URL", + "source_name": "stopforumspam", + "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole_filtered_50000.txt" + } + }, + { + "node": { + "external_id": null, + "source_name": "MISC", + "url": "https://example.com/CVE-0079-1234", + "description": null + } + } + ] + }, + "id": "fcfa872e-a8b6-4525-847e-f3c756b70035", + "is_inferred": false, + "killChainPhases": { + "edges": null + }, + "lang": "en", + "modified": "2023-11-09T23:22:20.586Z", + "name": "freelifetimexxxdates.com", + "objectLabel": { + "edges": [ + { + "node": { + "value": "spam" + } + } + ] + }, + "objectMarking": { + "edges": null + }, + "observables": { + "edges": [ + { + "node": { + "entity_type": "Domain-Name", + "id": "cc34949a-5a6f-4595-afec-c3bf98c62a7d", + "observable_value": "freelifetimexxxdates.com", + "standard_id": "domain-name--726e8863-8941-5a1b-b345-1f0131902233", + "value": "freelifetimexxxdates.com" + } + } + ], + "pageInfo": { + "globalCount": 1 + } + }, + "pattern": "[domain-name:value = 'freelifetimexxxdates.com']", + "pattern_type": "stix", + "revoked": false, + "standard_id": "indicator--08a7e875-2ce4-50ab-a8de-2915addd93c4", + "valid_from": "2023-11-09T23:22:19.426Z", + "valid_until": "2024-11-08T23:22:19.426Z", + "x_opencti_detection": false, + "x_opencti_main_observable_type": "Domain-Name", + "x_opencti_score": 60 + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json-expected.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json-expected.json new file mode 100644 index 00000000000..95f07ae5f30 --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json-expected.json @@ -0,0 +1,88 @@ +{ + "expected": [ + { + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2023-11-02T00:17:00.295Z", + "id": "fcfa872e-a8b6-4525-847e-f3c756b70035", + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "opencti": { + "indicator": { + "creator_identity_class": "organization", + "detection": false, + "external_reference": { + "description": "Stopforumspam feed URL", + "source_name": [ + "stopforumspam", + "MISC" + ], + "url": [ + "https://www.stopforumspam.com/downloads/toxic_domains_whole_filtered_50000.txt", + "https://example.com/CVE-0079-1234" + ] + }, + "is_inferred": false, + "lang": "en", + "observables_count": 1, + "pattern": "[domain-name:value = 'freelifetimexxxdates.com']", + "pattern_type": "stix", + "revoked": false, + "score": 60, + "standard_id": "indicator--08a7e875-2ce4-50ab-a8de-2915addd93c4", + "valid_from": "2023-11-09T23:22:19.426Z", + "valid_until": "2024-11-08T23:22:19.426Z" + }, + "observable": { + "domain_name": { + "entity_type": "Domain-Name", + "id": "cc34949a-5a6f-4595-afec-c3bf98c62a7d", + "standard_id": "domain-name--726e8863-8941-5a1b-b345-1f0131902233", + "value": "freelifetimexxxdates.com" + } + } + }, + "related": { + "hosts": [ + "freelifetimexxxdates.com" + ] + }, + "tags": [ + "forwarded", + "opencti-indicator", + "spam", + "ecs-indicator-detail" + ], + "threat": { + "feed": { + "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd", + "description": "Indicator data from OpenCTI", + "name": "OpenCTI", + "reference": "https://docs.opencti.io/latest/usage/overview/" + }, + "indicator": { + "confidence": "None", + "description": "Stopforumspam", + "modified_at": "2023-11-09T23:22:20.586Z", + "name": "freelifetimexxxdates.com", + "provider": "Stopforumspam", + "reference": "https://demo.opencti.io/dashboard/observations/indicators/fcfa872e-a8b6-4525-847e-f3c756b70035", + "type": "domain-name", + "url": { + "domain": "freelifetimexxxdates.com", + "registered_domain": "freelifetimexxxdates.com", + "top_level_domain": "com" + } + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 90094a42a66..015d9645c7e 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -146,7 +146,10 @@ processors: ignore_missing: true - gsub: - if: '!ctx.objectMarking.edges.isEmpty() && ctx.objectMarking.edges[0].node.definition_type == "TLP"' + if: | + ctx.objectMarking.edges != null && + !ctx.objectMarking.edges.isEmpty() && + ctx.objectMarking.edges[0].node.definition_type == "TLP" field: objectMarking.edges.0.node.definition pattern: '^TLP:' replacement: '' @@ -156,6 +159,7 @@ processors: - foreach: field: objectLabel.edges + ignore_missing: true processor: append: field: tags @@ -166,6 +170,7 @@ processors: - foreach: field: killChainPhases.edges + ignore_missing: true processor: append: field: opencti.indicator.kill_chain_phase @@ -173,12 +178,34 @@ processors: - remove: field: killChainPhases - - foreach: - field: externalReferences.edges - processor: - append: - field: opencti.indicator.external_reference - value: "{{{_ingest._value.node}}}" + - script: + description: Move external reference nodes to their target location, without nulls + lang: painless + source: | + ArrayList edges = ctx.externalReferences.edges; + if (edges == null) { + return; + } + for (int i = 0; i < edges.length; i++) { + if (!ctx.opencti?.indicator?.containsKey('external_reference') == true) { + if (!ctx.containsKey('opencti')) { + ctx.opencti = [:]; + } + if (!ctx.opencti.containsKey('indicator')) { + ctx.opencti.indicator = [:]; + } + if (!ctx.opencti.indicator.containsKey('external_reference')) { + ctx.opencti.indicator.external_reference = []; + } + } + def newNode = [:]; + for (def key : edges[i]['node'].keySet()) { + if (edges[i]['node'][key] != null) { + newNode[key] = edges[i]['node'][key]; + } + } + ctx.opencti.indicator.external_reference.add(newNode); + } - remove: field: externalReferences @@ -389,7 +416,7 @@ processors: return obj; } } - if (ctx.opencti?.containsKey('observable')) { + if (ctx.opencti?.containsKey('observable') == true) { ctx.opencti.observable = dropNulls(ctx.opencti.observable); for (def key : ctx.opencti.observable.keySet()) { if (ctx.opencti.observable[key].size() == 0) { @@ -754,6 +781,7 @@ processors: ############################################## # Restructure to avoid non-leaf lists, under # # - opencti.observable.* # + # - opencti.indicator.external_reference # # - threat.indicator.as # # - threat.indicator.file # # - threat.indicator.registry # @@ -795,11 +823,14 @@ processors: } return merged; } - if (ctx.opencti?.containsKey('observable')) { + if (ctx.opencti?.containsKey('observable') == true) { for (def key : ctx.opencti.observable.keySet()) { ctx.opencti.observable[key] = mergeListOfMaps(ctx.opencti.observable[key]); } } + if (ctx.opencti?.indicator?.containsKey('external_reference') == true) { + ctx.opencti.indicator.external_reference = mergeListOfMaps(ctx.opencti.indicator.external_reference); + } if (ctx.threat.indicator.containsKey('file')) { ctx.threat.indicator.file = mergeListOfMaps(ctx.threat.indicator.file); } diff --git a/packages/ti_opencti/manifest.yml b/packages/ti_opencti/manifest.yml index 1a456153d1c..7034b80d1c9 100644 --- a/packages/ti_opencti/manifest.yml +++ b/packages/ti_opencti/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.0" name: ti_opencti title: OpenCTI -version: "0.3.1" +version: "0.3.2" description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent." type: integration source: