Skip to content

Commit

Permalink
o365: Support IPv4 enclosed in square brackets (#18591)
Browse files Browse the repository at this point in the history
The O365 Management API is generating some events that contain a
`ClientIP` / `ClientIPAddress` field consisting of an IPv4 address
enclosed in square brackets. This is breaking ingestion of those events
as the brackets are only stripped for IPv6 addresses.

> "ClientIP": "[10.11.12.13]:12345"

This patch adds support for IPv4 enclosed in brackets and a few other
edge cases.

Fixes #18587
  • Loading branch information
adriansr authored May 16, 2020
1 parent bd7414d commit eaf196d
Show file tree
Hide file tree
Showing 4 changed files with 419 additions and 10 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ field. You can revert this change by configuring tags for the module and omittin
- Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]
- Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
- Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472]
- Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591]

*Heartbeat*

Expand Down
24 changes: 14 additions & 10 deletions x-pack/filebeat/module/o365/audit/config/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -725,22 +725,23 @@ function AuditProcessor(tenant_names, debug) {
'Yammer': yammerSchema(debug).Run,
}));

builder.Add("extractClientIPv4Port", new processor.Dissect({
tokenizer: '%{ip}:%{port}',
builder.Add("extractClientIPPortBrackets", new processor.Dissect({
tokenizer: '[%{_ip}]:%{port}',
field: 'client.address',
target_prefix: 'client',
'when.and': [
{'contains.client.address': '.'},
{'contains.client.address': ':'},
],
'when.contains.client.address': ']:',
}));
builder.Add("extractClientIPv6Port", new processor.Dissect({
tokenizer: '[%{ip}]:%{port}',
builder.Add("extractClientIPv4Port", new processor.Dissect({
tokenizer: '%{_ip}:%{port}',
field: 'client.address',
target_prefix: 'client',
'when.and': [
{'contains.client.address': '['},
{'not.has_fields': ['client._ip', 'client.port']},
{'contains.client.address': '.'},
{'contains.client.address': ':'},
// Best effort to avoid parsing IPv6-mapped IPv4 as ip:port.
// Won't succeed if IPv6 address is not shortened.
{'not.contains.client.address': '::'},
],
}));

Expand All @@ -749,11 +750,14 @@ function AuditProcessor(tenant_names, debug) {
fields: [
{from: "client.address", to: "client.ip", type: "ip"},
{from: "server.address", to: "server.ip", type: "ip"},
{from: "client._ip", to: "client.ip", type: "ip"},
],
ignore_missing: true,
fail_on_error: false
}));

builder.Add("removeTempIP", function (evt) {
evt.Delete("client._ip");
});
builder.Add("setSrcDstFields", new processor.Convert({
fields: [
{from: "client.ip", to: "source.ip"},
Expand Down
15 changes: 15 additions & 0 deletions x-pack/filebeat/module/o365/audit/test/ip-formats.log
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{"ClientIP":"[10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"10.11.12.13:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"::ffff:10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"[::ffff:10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"[2001:db8::abcd]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"2001:db8::abcd","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"[2001:db8::abcd]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"[10.11.12.13]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"localhost","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"[localhost]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"localhost:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"[cool.client.local]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"cool.client.local","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
{"ClientIP":"cool.client.local:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
Loading

0 comments on commit eaf196d

Please sign in to comment.