From d61819f39bad79cd30c8588a7cd5d62fc831d76c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 15 Jun 2020 12:06:51 +0200 Subject: [PATCH] Support up to 7 bytepad for Netflow IPFIX template (#19131) Fixes #18098 (cherry picked from commit fb94175714db0dc2638aa50e3c7136b5e7b4d5a2) --- CHANGELOG.next.asciidoc | 1 + .../input/netflow/decoder/v9/decoder.go | 2 +- ...netflow9_e10s_4_7byte_pad.pcap.golden.json | 461 ++++++++++++++++++ .../pcap/netflow9_e10s_4_7byte_pad.pcap | Bin 0 -> 838 bytes 4 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json create mode 100644 x-pack/filebeat/input/netflow/testdata/pcap/netflow9_e10s_4_7byte_pad.pcap diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2a7295897df..fb3a9db1fab 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -209,6 +209,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630] - Fix `o365` module ignoring `var.api` settings. {pull}18948[18948] - Okta module now sets the Elasticsearch `_id` field to the Okta UUID value contained in each system log to minimize the possibility of duplicating events. {pull}18953[18953] +- Fix `netflow` module to support 7 bytepad for IPFIX template. {issue}18098[18098] *Heartbeat* diff --git a/x-pack/filebeat/input/netflow/decoder/v9/decoder.go b/x-pack/filebeat/input/netflow/decoder/v9/decoder.go index 4901ba7d6a8..da82fbc1225 100644 --- a/x-pack/filebeat/input/netflow/decoder/v9/decoder.go +++ b/x-pack/filebeat/input/netflow/decoder/v9/decoder.go @@ -137,7 +137,7 @@ func ReadFields(d Decoder, buf *bytes.Buffer, count int) (record template.Templa func ReadTemplateFlowSet(d Decoder, buf *bytes.Buffer) (templates []*template.Template, err error) { var row [4]byte for { - if buf.Len() < 4 { + if buf.Len() < 8 { return templates, nil } if n, err := buf.Read(row[:]); err != nil || n != len(row) { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json new file mode 100644 index 00000000000..a71c8fb7ba8 --- /dev/null +++ b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json @@ -0,0 +1,461 @@ +{ + "test_name": "netflow9_e10s_4_7byte_pad", + "events": [ + { + "Timestamp": "2020-04-16T23:22:51Z", + "Meta": null, + "Fields": { + "destination": { + "ip": "10.36.236.100", + "locality": "private", + "port": 54594 + }, + "event": { + "action": "netflow_flow", + "category": [ + "network_traffic", + "network" + ], + "created": "2020-04-16T23:22:51Z", + "kind": "event", + "type": [ + "connection" + ] + }, + "flow": { + "id": "6mUV1nPVG80", + "locality": "private" + }, + "netflow": { + "destination_ipv4_address": "10.36.236.100", + "destination_transport_port": 54594, + "egress_interface": 1, + "exporter": { + "address": "109.180.55.123:10000", + "source_id": 1, + "timestamp": "2020-04-16T23:22:51Z", + "uptime_millis": 0, + "version": 10 + }, + "flow_end_milliseconds": "2020-04-16T23:22:48.963Z", + "flow_end_reason": 3, + "flow_start_milliseconds": "2020-04-16T23:22:48.96Z", + "ingress_interface": 1, + "octet_delta_count": 1855, + "packet_delta_count": 5, + "protocol_identifier": 6, + "source_ipv4_address": "10.127.32.11", + "source_transport_port": 53, + "tcp_control_bits": 27, + "type": "netflow_flow" + }, + "network": { + "bytes": 1855, + "community_id": "1:+/kh1SKruHHnZ5JGSMfWk9nZx8o=", + "direction": "unknown", + "iana_number": 6, + "packets": 5, + "transport": "tcp" + }, + "observer": { + "ip": "109.180.55.123" + }, + "related": { + "ip": [ + "10.127.32.11", + "10.36.236.100" + ] + }, + "source": { + "bytes": 1855, + "ip": "10.127.32.11", + "locality": "private", + "packets": 5, + "port": 53 + } + }, + "Private": null, + "TimeSeries": false + }, + { + "Timestamp": "2020-04-16T23:22:51Z", + "Meta": null, + "Fields": { + "destination": { + "ip": "10.36.237.22", + "locality": "private", + "port": 52058 + }, + "event": { + "action": "netflow_flow", + "category": [ + "network_traffic", + "network" + ], + "created": "2020-04-16T23:22:51Z", + "kind": "event", + "type": [ + "connection" + ] + }, + "flow": { + "id": "3BTOVt9gp8I", + "locality": "private" + }, + "netflow": { + "destination_ipv4_address": "10.36.237.22", + "destination_transport_port": 52058, + "egress_interface": 1, + "exporter": { + "address": "109.180.55.123:10000", + "source_id": 1, + "timestamp": "2020-04-16T23:22:51Z", + "uptime_millis": 0, + "version": 10 + }, + "flow_end_milliseconds": "2020-04-16T23:22:48.901Z", + "flow_end_reason": 3, + "flow_start_milliseconds": "2020-04-16T23:22:48.9Z", + "ingress_interface": 1, + "octet_delta_count": 217, + "packet_delta_count": 3, + "protocol_identifier": 6, + "source_ipv4_address": "10.36.228.103", + "source_transport_port": 8000, + "tcp_control_bits": 25, + "type": "netflow_flow" + }, + "network": { + "bytes": 217, + "community_id": "1:FAOWMcPTJlyjuohaFfnr9oyvnIo=", + "direction": "unknown", + "iana_number": 6, + "packets": 3, + "transport": "tcp" + }, + "observer": { + "ip": "109.180.55.123" + }, + "related": { + "ip": [ + "10.36.228.103", + "10.36.237.22" + ] + }, + "source": { + "bytes": 217, + "ip": "10.36.228.103", + "locality": "private", + "packets": 3, + "port": 8000 + } + }, + "Private": null, + "TimeSeries": false + }, + { + "Timestamp": "2020-04-16T23:22:51Z", + "Meta": null, + "Fields": { + "destination": { + "ip": "10.127.32.11", + "locality": "private", + "port": 53 + }, + "event": { + "action": "netflow_flow", + "category": [ + "network_traffic", + "network" + ], + "created": "2020-04-16T23:22:51Z", + "kind": "event", + "type": [ + "connection" + ] + }, + "flow": { + "id": "6mUV1nPVG80", + "locality": "private" + }, + "netflow": { + "destination_ipv4_address": "10.127.32.11", + "destination_transport_port": 53, + "egress_interface": 1, + "exporter": { + "address": "109.180.55.123:10000", + "source_id": 1, + "timestamp": "2020-04-16T23:22:51Z", + "uptime_millis": 0, + "version": 10 + }, + "flow_end_milliseconds": "2020-04-16T23:22:48.963Z", + "flow_end_reason": 3, + "flow_start_milliseconds": "2020-04-16T23:22:48.96Z", + "ingress_interface": 1, + "octet_delta_count": 547, + "packet_delta_count": 7, + "protocol_identifier": 6, + "source_ipv4_address": "10.36.236.100", + "source_transport_port": 54594, + "tcp_control_bits": 27, + "type": "netflow_flow" + }, + "network": { + "bytes": 547, + "community_id": "1:+/kh1SKruHHnZ5JGSMfWk9nZx8o=", + "direction": "unknown", + "iana_number": 6, + "packets": 7, + "transport": "tcp" + }, + "observer": { + "ip": "109.180.55.123" + }, + "related": { + "ip": [ + "10.36.236.100", + "10.127.32.11" + ] + }, + "source": { + "bytes": 547, + "ip": "10.36.236.100", + "locality": "private", + "packets": 7, + "port": 54594 + } + }, + "Private": null, + "TimeSeries": false + }, + { + "Timestamp": "2020-04-16T23:22:51Z", + "Meta": null, + "Fields": { + "destination": { + "ip": "10.36.236.100", + "locality": "private", + "port": 49180 + }, + "event": { + "action": "netflow_flow", + "category": [ + "network_traffic", + "network" + ], + "created": "2020-04-16T23:22:51Z", + "kind": "event", + "type": [ + "connection" + ] + }, + "flow": { + "id": "HVg4SttTufc", + "locality": "public" + }, + "netflow": { + "destination_ipv4_address": "10.36.236.100", + "destination_transport_port": 49180, + "egress_interface": 1, + "exporter": { + "address": "109.180.55.123:10000", + "source_id": 1, + "timestamp": "2020-04-16T23:22:51Z", + "uptime_millis": 0, + "version": 10 + }, + "flow_end_milliseconds": "2020-04-16T23:22:48.404Z", + "flow_end_reason": 3, + "flow_start_milliseconds": "2020-04-16T23:22:47.995Z", + "ingress_interface": 1, + "octet_delta_count": 7158, + "packet_delta_count": 10, + "protocol_identifier": 6, + "source_ipv4_address": "52.206.251.4", + "source_transport_port": 443, + "tcp_control_bits": 27, + "type": "netflow_flow" + }, + "network": { + "bytes": 7158, + "community_id": "1:Zyly7BCJ6D7luuRJJazRxZ/mFZM=", + "direction": "unknown", + "iana_number": 6, + "packets": 10, + "transport": "tcp" + }, + "observer": { + "ip": "109.180.55.123" + }, + "related": { + "ip": [ + "52.206.251.4", + "10.36.236.100" + ] + }, + "source": { + "bytes": 7158, + "ip": "52.206.251.4", + "locality": "public", + "packets": 10, + "port": 443 + } + }, + "Private": null, + "TimeSeries": false + }, + { + "Timestamp": "2020-04-16T23:22:51Z", + "Meta": null, + "Fields": { + "destination": { + "ip": "52.206.251.4", + "locality": "public", + "port": 443 + }, + "event": { + "action": "netflow_flow", + "category": [ + "network_traffic", + "network" + ], + "created": "2020-04-16T23:22:51Z", + "kind": "event", + "type": [ + "connection" + ] + }, + "flow": { + "id": "HVg4SttTufc", + "locality": "public" + }, + "netflow": { + "destination_ipv4_address": "52.206.251.4", + "destination_transport_port": 443, + "egress_interface": 1, + "exporter": { + "address": "109.180.55.123:10000", + "source_id": 1, + "timestamp": "2020-04-16T23:22:51Z", + "uptime_millis": 0, + "version": 10 + }, + "flow_end_milliseconds": "2020-04-16T23:22:48.404Z", + "flow_end_reason": 3, + "flow_start_milliseconds": "2020-04-16T23:22:47.92Z", + "ingress_interface": 1, + "octet_delta_count": 1538, + "packet_delta_count": 11, + "protocol_identifier": 6, + "source_ipv4_address": "10.36.236.100", + "source_transport_port": 49180, + "tcp_control_bits": 27, + "type": "netflow_flow" + }, + "network": { + "bytes": 1538, + "community_id": "1:Zyly7BCJ6D7luuRJJazRxZ/mFZM=", + "direction": "unknown", + "iana_number": 6, + "packets": 11, + "transport": "tcp" + }, + "observer": { + "ip": "109.180.55.123" + }, + "related": { + "ip": [ + "10.36.236.100", + "52.206.251.4" + ] + }, + "source": { + "bytes": 1538, + "ip": "10.36.236.100", + "locality": "private", + "packets": 11, + "port": 49180 + } + }, + "Private": null, + "TimeSeries": false + }, + { + "Timestamp": "2020-04-16T23:22:51Z", + "Meta": null, + "Fields": { + "destination": { + "ip": "10.36.228.103", + "locality": "private", + "port": 8000 + }, + "event": { + "action": "netflow_flow", + "category": [ + "network_traffic", + "network" + ], + "created": "2020-04-16T23:22:51Z", + "kind": "event", + "type": [ + "connection" + ] + }, + "flow": { + "id": "3BTOVt9gp8I", + "locality": "private" + }, + "netflow": { + "destination_ipv4_address": "10.36.228.103", + "destination_transport_port": 8000, + "egress_interface": 1, + "exporter": { + "address": "109.180.55.123:10000", + "source_id": 1, + "timestamp": "2020-04-16T23:22:51Z", + "uptime_millis": 0, + "version": 10 + }, + "flow_end_milliseconds": "2020-04-16T23:22:48.901Z", + "flow_end_reason": 3, + "flow_start_milliseconds": "2020-04-16T23:22:48.9Z", + "ingress_interface": 1, + "octet_delta_count": 217, + "packet_delta_count": 3, + "protocol_identifier": 6, + "source_ipv4_address": "10.36.237.22", + "source_transport_port": 52058, + "tcp_control_bits": 25, + "type": "netflow_flow" + }, + "network": { + "bytes": 217, + "community_id": "1:FAOWMcPTJlyjuohaFfnr9oyvnIo=", + "direction": "unknown", + "iana_number": 6, + "packets": 3, + "transport": "tcp" + }, + "observer": { + "ip": "109.180.55.123" + }, + "related": { + "ip": [ + "10.36.237.22", + "10.36.228.103" + ] + }, + "source": { + "bytes": 217, + "ip": "10.36.237.22", + "locality": "private", + "packets": 3, + "port": 52058 + } + }, + "Private": null, + "TimeSeries": false + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/input/netflow/testdata/pcap/netflow9_e10s_4_7byte_pad.pcap b/x-pack/filebeat/input/netflow/testdata/pcap/netflow9_e10s_4_7byte_pad.pcap new file mode 100644 index 0000000000000000000000000000000000000000..f85c4bbbd13259a59a48f2d750f5ee950fbc529e GIT binary patch literal 838 zcmb7CPbh<782`QBKHuisEUZM|LRmR0a&WL#N|?qDrWv+pbWtdysIJ%4(B@ALki_r1L+9B9z-V>%R*QZ>8yany_g zV*$ETD>rFatHJT44`aY#Xa2rH^;AMhmvv9;U~DrOqC9`|__DGxEH|dL3`#%gyumIy}Gd9(L84aD*%MG8HHT zIAttoP>>JJ))Y*;6lB?HVvh>SRPe}5K|_?+XklGPkjTP^9H9t}$S{(%8O*$dzgE=| za26|k^-sQqR)h(fD9H^3u+K{fek)Pip(OYg)<(S&wZ=|UE!QhiQ1h(T?UavLdy5$l z1Dbc#_GprwVyF@`)4l%2w@6&tyu_8$f9H4)d0jox@`4$8N|}+D4%Yver}a5Jugt5& zFeI~NrkfWZvFAdnnFG>2FFGsUp|XN2pX_qV?CLQhn* IiAFi^8;RAAc>n+a literal 0 HcmV?d00001