[filebeat][threatintel] MISP splitting fix for empty responses (#38917)

Two fixes related to empty server responses: - Set `response.split.ignore_empty_value` to `true` to avoid indexing `{response:[]}`, which is the MISP server's empty response body. According to the `response.split` [documentation][1], "If the split target is empty the parent document will be kept. If documents with empty splits should be dropped, the `ignore_empty_value` option should be set to `true`." - Use the null-safe operator for a chained method invocation following a null-safe field access. All other null-safe operator usages were also reviewed and corrected where necessary. [1]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-split --------- Co-authored-by: Dan Kortschak <[email protected]> (cherry picked from commit 692658c)
elastic · Apr 15, 2024 · f2e14f8 · f2e14f8
1 parent 0c369e3
commit f2e14f8
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 18 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -49,6 +49,21 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 - Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094]
 - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116]
+- Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125]
+- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181]
+- Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216]
+- Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290]
+- Fix PEM key validation for CEL and HTTPJSON inputs. {pull}38405[38405]
+- Fix filebeat gcs input panic {pull}38407[38407]
+- Rename `activity_guid` to `activity_id` in ETW input events to suit other Windows inputs. {pull}38530[38530]
+- Add missing provider registration and fix published entity for Active Directory entityanalytics provider. {pull}38645[38645]
+- Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709]
+- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488]
+- Fix indexing failures by re-enabling event normalisation in netflow input. {issue}38703[38703] {pull}38780[38780]
+- Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416]
+- Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556]
+- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488]
+- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml
@@ -49,6 +49,7 @@ request.transforms:
 
 response.split:
   target: body.response
+  ignore_empty_value: true
   split:
     target: body.Event.Attribute
     ignore_empty_value: true

diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml
@@ -63,12 +63,12 @@ processors:
       field:
         - misp.Attribute
       ignore_missing: true
-      if: ctx.misp?.Attribute.size() == 0
+      if: ctx.misp?.Attribute?.size() == 0
   - remove:
       field:
         - misp.Object
       ignore_missing: true
-      if: ctx.misp?.Object.size() == 0
+      if: ctx.misp?.Object?.size() == 0
   - date:
       field: misp.timestamp
       formats:
@@ -144,12 +144,12 @@ processors:
   - set:
       field: threat.indicator.type
       value: file
-      if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))"
+      if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type?.startsWith('filename') == true"
   - rename:
       field: misp.attribute.value
       target_field: "threat.indicator.file.hash.{{misp.attribute.type}}"
       ignore_missing: true
-      if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')"
+      if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.file.name
@@ -160,23 +160,23 @@ processors:
       patterns:
         - "%{WORD}\\|%{WORD:_tmp.hashtype}"
       ignore_missing: true
-      if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
+      if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
   - grok:
       field: misp.attribute.value
       patterns:
         - "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
       ignore_missing: true
-      if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
+      if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
   - set:
       field: threat.indicator.file.hash.{{_tmp.hashtype}}
       value: "{{_tmp.hashvalue}}"
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
+      if: "ctx.misp?.attribute?.type?.startsWith('filename|') == true && ctx._tmp?.hashvalue != null && ctx._tmp?.hashtype != null"
 
   ## URL/URI indicator operations
   - set:
       field: threat.indicator.type
       value: url
-      if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
+      if: "['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
   - uri_parts:
       field: misp.attribute.value
       target_field: threat.indicator.url
@@ -193,7 +193,7 @@ processors:
   - set:
       field: threat.indicator.type
       value: windows-registry-key
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')"
+      if: "ctx.misp?.attribute?.type?.startsWith('regkey') == true"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.registry.key
@@ -210,7 +210,7 @@ processors:
   - set:
       field: threat.indicator.type
       value: autonomous-system
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'"
+      if: "ctx.misp?.attribute?.type == 'AS'"
   - convert:
       field: misp.attribute.value
       type: long
@@ -222,11 +222,11 @@ processors:
   - set:
       field: threat.indicator.type
       value: domain-name
-      if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))"
+      if: "ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type?.startsWith('domain') == true"
   - set:
       field: threat.indicator.type
       value: ipv4-addr
-      if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
+      if: "['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.url.domain
@@ -257,11 +257,11 @@ processors:
   - set:
       field: threat.indicator.type
       value: email-addr
-      if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
+      if: "['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
   - set:
       field: threat.indicator.type
       value: email-message
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
+      if: "ctx.misp?.attribute?.type?.startsWith('email') == true && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.email.address
@@ -274,13 +274,13 @@ processors:
   - append:
       field: user.roles
       value: "reporting_user"
-      if: ctx?.user?.email != null
+      if: ctx.user?.email != null
 
   ## MAC Address indicator operations
   - set:
       field: threat.indicator.type
       value: mac-addr
-      if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
+      if: "['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.mac
@@ -337,12 +337,12 @@ processors:
   ######################
   - remove:
       field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
       ignore_failure: true
       ignore_missing: true
   - script:
       lang: painless
-      if: ctx?.misp != null
+      if: ctx.misp != null
       source: |
         void handleMap(Map map) {
           for (def x : map.values()) {