Skip to content

Commit

Permalink
[Filebeat] Improve ECS categorization field mappings for zeek module (e…
Browse files Browse the repository at this point in the history 
…lastic#17738)

* Improve ECS categorization field mappings for zeek module

- capture_loss
  + convert pipeline to yaml
  + event.kind
  + event.type
- connection
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
- dce_rpc
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + related.ip
  + source.geo
  + destination.geo
- dhcp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
- dnp3
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + related.ip
  + source.geo
  + destination.geo
  + source.as
  + destiantion.as
- dns
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + dns.question.top_level_domain
  + related.ip
- dpd
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + source.geo
  + destiantion.geo
  + source.as
  + destiantion.as
  + related.ip
- files
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + file.mime_type
  + file.name
  + file.hash.md5
  + file.hash.sha1
  + file.hash.sha256
  + client.ip
  + server.ip
  + related.ip
  + related.hash
- ftp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + user.name
  + file.mime_type
  + file.size
  + related.ip
  + related.user
  + source.geo
  + destination.geo
- http
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + url.port type to number
  + http.request.method lowercase
  + related.ip
  + related.users
- intel
  + event.kind
  + event.type
  + related.ip
- irc
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + user.name
  + file.name
  + file.size
  + file.mime_type
  + related.ip
  + related.user
  + source.geo
  + destination.geo
- kerberos
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + user.name
  + user.domain
  + source.geo
  + destination.geo
  + related.ip
  + related.user
- modbus
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + related.ip
  + source.geo
  + destination.geo
- mysql
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + source.geo
  + destiantion.geo
  + related.ip
- notice
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + file.size
  + file.mime_type
  + rule.description
  + rule.name
- ntlm
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + user.name
  + user.domain
  + source.geo
  + destination.geo
  + related.ip
  + related.user
- ocsp
  + convert pipeline to yaml
  + event.kind
  + related.hash
- pe
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
- radius
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + user.name
  + source.geo
  + destination.geo
  + related.ip
  + related.user
- rdp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- rfb
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- sip
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + related.ip
  + source.geo
  + destination.geo
  + url.full
- smb_cmd
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + event.action
  + related.ip
  + related.user
  + source.geo
  + destination.geo
  + user.name
- smb_files
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.action
  + related.ip
  + related.user
  + source.geo
  + destination.geo
  + user.name
  + file.accessed
  + file.ctime
  + file.created
  + file.mtime
  + file.path
  + file.name
  + file.size
- smb_mapping
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- smtp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- snmp
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- socks
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + related.ip
  + source.geo
  + destination.geo
  + user.name
  + related.user
- ssh
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + event.outcome
  + related.ip
  + source.geo
  + destination.geo
- ssl
  + event.kind
  + event.category
  + event.type
  + related.ip
- stats
  + convert pipeline to yaml
  + event.kind
- syslog
  + convert pipeline to yaml
  + event.kind
  + related.ip
  + source.geo
  + destination.geo
  + log.syslog.facility.name
  + log.syslog.severity.name
- traceroute
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- tunnel
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
- weird
  + convert pipeline to yaml
  + event.kind
  + event.category
  + event.type
  + related.ip
  + source.geo
  + destination.geo
  + rule.name
- x509
  + event.kind
  + event.type

Closes elastic#16029
  • Loading branch information
@leehinman
leehinman authored Apr 29, 2020
1 parent 7d16a0f commit b552dd8
Show file tree
Hide file tree
Showing 172 changed files with 3,728 additions and 2,113 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
- Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
- Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]
- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]

*Heartbeat*

Expand Down
28 changes: 0 additions & 28 deletions x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.json
View file

This file was deleted.

21 changes: 21 additions & 0 deletions x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
description: Pipeline for normalizing Zeek capture_loss.log
processors:
- set:
field: event.created
value: '{{_ingest.timestamp}}'
- date:
field: zeek.capture_loss.ts
formats:
- UNIX
- remove:
field: zeek.capture_loss.ts
- set:
field: event.kind
value: metric
- set:
field: event.type
value: info
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/zeek/capture_loss/manifest.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ var:
- name: tags
default: [zeek.capture_loss]

ingest_pipeline: ingest/pipeline.json
ingest_pipeline: ingest/pipeline.yml
input: config/capture_loss.yml
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/zeek/capture_loss/test/capture_loss-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
{
"@timestamp": "2019-09-10T16:19:28.465Z",
"event.dataset": "zeek.capture_loss",
"event.kind": "metric",
"event.module": "zeek",
"event.type": "info",
"fileset.name": "capture_loss",
"input.type": "log",
"log.offset": 0,
Expand Down
19 changes: 13 additions & 6 deletions x-pack/filebeat/module/zeek/connection/config/connection.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,20 +75,27 @@ processors:

ignore_missing: true
fail_on_error: false

- convert:
fields:
- {from: "zeek.session_id", to: "event.id"}
- {from: "source.address", to: "source.ip", type: "ip"}
- {from: "destination.address", to: "destination.ip", type: "ip"}
ignore_missing: true
fail_on_error: false
- add_fields:
target: event
fields:
kind: event
category:
- network
{{ if .community_id }}
- if:
equals.network.transport: icmp
then:
community_id:
fields:
source_ip: source.address
destination_ip: destination.address
icmp_type: zeek.connection.icmp.type
icmp_code: zeek.connection.icmp.code
else:
community_id:
fields:
source_ip: source.address
destination_ip: destination.address
{{ end }}
160 changes: 0 additions & 160 deletions x-pack/filebeat/module/zeek/connection/ingest/pipeline.json
View file

This file was deleted.

Loading

0 comments on commit b552dd8

Please sign in to comment.