Skip to content

Commit 5a5ac23

Browse files
chrisberkhoutchrisberkhout
authored
[ti_opencti] Fix processing of externalReferences (#8556)
The contents of externalReferences.edges were being processed with the append processor and mistakenly flattened into scalar values. To fix that they are now handled in Painless, as objects. Additional checks were added for collection fields that may be null, after some were observed in example data added to test the externalReferences fix. Fixed null handling in if conditions.
1 parent 0663a7e commit 5a5ac23

File tree

5 files changed

+211
-10
lines changed

5 files changed

+211
-10
lines changed

‎packages/ti_opencti/changelog.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "0.3.2"
3+
changes:
4+
- description: Fix processing of externalReferences.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/8556
27
- version: "0.3.1"
38
changes:
49
- description: Fix event.original field type conflict

‎packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
{
2+
"events": [
3+
{
4+
"confidence": 0,
5+
"created": "2023-11-02T00:17:00.295Z",
6+
"createdBy": {
7+
"identity_class": "organization",
8+
"name": "Stopforumspam"
9+
},
10+
"description": "Stopforumspam",
11+
"externalReferences": {
12+
"edges": [
13+
{
14+
"node": {
15+
"description": "Stopforumspam feed URL",
16+
"source_name": "stopforumspam",
17+
"url": "https://www.stopforumspam.com/downloads/toxic_domains_whole_filtered_50000.txt"
18+
}
19+
},
20+
{
21+
"node": {
22+
"external_id": null,
23+
"source_name": "MISC",
24+
"url": "https://example.com/CVE-0079-1234",
25+
"description": null
26+
}
27+
}
28+
]
29+
},
30+
"id": "fcfa872e-a8b6-4525-847e-f3c756b70035",
31+
"is_inferred": false,
32+
"killChainPhases": {
33+
"edges": null
34+
},
35+
"lang": "en",
36+
"modified": "2023-11-09T23:22:20.586Z",
37+
"name": "freelifetimexxxdates.com",
38+
"objectLabel": {
39+
"edges": [
40+
{
41+
"node": {
42+
"value": "spam"
43+
}
44+
}
45+
]
46+
},
47+
"objectMarking": {
48+
"edges": null
49+
},
50+
"observables": {
51+
"edges": [
52+
{
53+
"node": {
54+
"entity_type": "Domain-Name",
55+
"id": "cc34949a-5a6f-4595-afec-c3bf98c62a7d",
56+
"observable_value": "freelifetimexxxdates.com",
57+
"standard_id": "domain-name--726e8863-8941-5a1b-b345-1f0131902233",
58+
"value": "freelifetimexxxdates.com"
59+
}
60+
}
61+
],
62+
"pageInfo": {
63+
"globalCount": 1
64+
}
65+
},
66+
"pattern": "[domain-name:value = 'freelifetimexxxdates.com']",
67+
"pattern_type": "stix",
68+
"revoked": false,
69+
"standard_id": "indicator--08a7e875-2ce4-50ab-a8de-2915addd93c4",
70+
"valid_from": "2023-11-09T23:22:19.426Z",
71+
"valid_until": "2024-11-08T23:22:19.426Z",
72+
"x_opencti_detection": false,
73+
"x_opencti_main_observable_type": "Domain-Name",
74+
"x_opencti_score": 60
75+
}
76+
]
77+
}

‎packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-domain-name-with-external-reference.json-expected.json

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
{
2+
"expected": [
3+
{
4+
"ecs": {
5+
"version": "8.11.0"
6+
},
7+
"event": {
8+
"category": [
9+
"threat"
10+
],
11+
"created": "2023-11-02T00:17:00.295Z",
12+
"id": "fcfa872e-a8b6-4525-847e-f3c756b70035",
13+
"kind": "enrichment",
14+
"type": [
15+
"indicator"
16+
]
17+
},
18+
"opencti": {
19+
"indicator": {
20+
"creator_identity_class": "organization",
21+
"detection": false,
22+
"external_reference": {
23+
"description": "Stopforumspam feed URL",
24+
"source_name": [
25+
"stopforumspam",
26+
"MISC"
27+
],
28+
"url": [
29+
"https://www.stopforumspam.com/downloads/toxic_domains_whole_filtered_50000.txt",
30+
"https://example.com/CVE-0079-1234"
31+
]
32+
},
33+
"is_inferred": false,
34+
"lang": "en",
35+
"observables_count": 1,
36+
"pattern": "[domain-name:value = 'freelifetimexxxdates.com']",
37+
"pattern_type": "stix",
38+
"revoked": false,
39+
"score": 60,
40+
"standard_id": "indicator--08a7e875-2ce4-50ab-a8de-2915addd93c4",
41+
"valid_from": "2023-11-09T23:22:19.426Z",
42+
"valid_until": "2024-11-08T23:22:19.426Z"
43+
},
44+
"observable": {
45+
"domain_name": {
46+
"entity_type": "Domain-Name",
47+
"id": "cc34949a-5a6f-4595-afec-c3bf98c62a7d",
48+
"standard_id": "domain-name--726e8863-8941-5a1b-b345-1f0131902233",
49+
"value": "freelifetimexxxdates.com"
50+
}
51+
}
52+
},
53+
"related": {
54+
"hosts": [
55+
"freelifetimexxxdates.com"
56+
]
57+
},
58+
"tags": [
59+
"forwarded",
60+
"opencti-indicator",
61+
"spam",
62+
"ecs-indicator-detail"
63+
],
64+
"threat": {
65+
"feed": {
66+
"dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd",
67+
"description": "Indicator data from OpenCTI",
68+
"name": "OpenCTI",
69+
"reference": "https://docs.opencti.io/latest/usage/overview/"
70+
},
71+
"indicator": {
72+
"confidence": "None",
73+
"description": "Stopforumspam",
74+
"modified_at": "2023-11-09T23:22:20.586Z",
75+
"name": "freelifetimexxxdates.com",
76+
"provider": "Stopforumspam",
77+
"reference": "https://demo.opencti.io/dashboard/observations/indicators/fcfa872e-a8b6-4525-847e-f3c756b70035",
78+
"type": "domain-name",
79+
"url": {
80+
"domain": "freelifetimexxxdates.com",
81+
"registered_domain": "freelifetimexxxdates.com",
82+
"top_level_domain": "com"
83+
}
84+
}
85+
}
86+
}
87+
]
88+
}

‎packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,10 @@ processors:
146146
ignore_missing: true
147147

148148
- gsub:
149-
if: '!ctx.objectMarking.edges.isEmpty() && ctx.objectMarking.edges[0].node.definition_type == "TLP"'
149+
if: |
150+
ctx.objectMarking.edges != null &&
151+
!ctx.objectMarking.edges.isEmpty() &&
152+
ctx.objectMarking.edges[0].node.definition_type == "TLP"
150153
field: objectMarking.edges.0.node.definition
151154
pattern: '^TLP:'
152155
replacement: ''
@@ -156,6 +159,7 @@ processors:
156159

157160
- foreach:
158161
field: objectLabel.edges
162+
ignore_missing: true
159163
processor:
160164
append:
161165
field: tags
@@ -166,19 +170,42 @@ processors:
166170

167171
- foreach:
168172
field: killChainPhases.edges
173+
ignore_missing: true
169174
processor:
170175
append:
171176
field: opencti.indicator.kill_chain_phase
172177
value: "[{{{_ingest._value.node.kill_chain_name}}}] {{{_ingest._value.node.phase_name}}}"
173178
- remove:
174179
field: killChainPhases
175180

176-
- foreach:
177-
field: externalReferences.edges
178-
processor:
179-
append:
180-
field: opencti.indicator.external_reference
181-
value: "{{{_ingest._value.node}}}"
181+
- script:
182+
description: Move external reference nodes to their target location, without nulls
183+
lang: painless
184+
source: |
185+
ArrayList edges = ctx.externalReferences.edges;
186+
if (edges == null) {
187+
return;
188+
}
189+
for (int i = 0; i < edges.length; i++) {
190+
if (!ctx.opencti?.indicator?.containsKey('external_reference') == true) {
191+
if (!ctx.containsKey('opencti')) {
192+
ctx.opencti = [:];
193+
}
194+
if (!ctx.opencti.containsKey('indicator')) {
195+
ctx.opencti.indicator = [:];
196+
}
197+
if (!ctx.opencti.indicator.containsKey('external_reference')) {
198+
ctx.opencti.indicator.external_reference = [];
199+
}
200+
}
201+
def newNode = [:];
202+
for (def key : edges[i]['node'].keySet()) {
203+
if (edges[i]['node'][key] != null) {
204+
newNode[key] = edges[i]['node'][key];
205+
}
206+
}
207+
ctx.opencti.indicator.external_reference.add(newNode);
208+
}
182209
- remove:
183210
field: externalReferences
184211

@@ -389,7 +416,7 @@ processors:
389416
return obj;
390417
}
391418
}
392-
if (ctx.opencti?.containsKey('observable')) {
419+
if (ctx.opencti?.containsKey('observable') == true) {
393420
ctx.opencti.observable = dropNulls(ctx.opencti.observable);
394421
for (def key : ctx.opencti.observable.keySet()) {
395422
if (ctx.opencti.observable[key].size() == 0) {
@@ -754,6 +781,7 @@ processors:
754781
##############################################
755782
# Restructure to avoid non-leaf lists, under #
756783
# - opencti.observable.* #
784+
# - opencti.indicator.external_reference #
757785
# - threat.indicator.as #
758786
# - threat.indicator.file #
759787
# - threat.indicator.registry #
@@ -795,11 +823,14 @@ processors:
795823
}
796824
return merged;
797825
}
798-
if (ctx.opencti?.containsKey('observable')) {
826+
if (ctx.opencti?.containsKey('observable') == true) {
799827
for (def key : ctx.opencti.observable.keySet()) {
800828
ctx.opencti.observable[key] = mergeListOfMaps(ctx.opencti.observable[key]);
801829
}
802830
}
831+
if (ctx.opencti?.indicator?.containsKey('external_reference') == true) {
832+
ctx.opencti.indicator.external_reference = mergeListOfMaps(ctx.opencti.indicator.external_reference);
833+
}
803834
if (ctx.threat.indicator.containsKey('file')) {
804835
ctx.threat.indicator.file = mergeListOfMaps(ctx.threat.indicator.file);
805836
}

‎packages/ti_opencti/manifest.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: "3.0.0"
22
name: ti_opencti
33
title: OpenCTI
4-
version: "0.3.1"
4+
version: "0.3.2"
55
description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent."
66
type: integration
77
source:

0 commit comments

Comments
 (0)