From 2ef3b95684f977f31c95b3acfa6028d783774fd9 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 6 Dec 2018 16:06:48 -0500 Subject: [PATCH 1/5] Introduce `source.address` and `destination.address`. --- README.md | 2 ++ fields.yml | 22 ++++++++++++++++++++++ schema.csv | 2 ++ schemas/destination.yml | 11 +++++++++++ schemas/source.yml | 11 +++++++++++ template.json | 8 ++++++++ 6 files changed, 56 insertions(+) diff --git a/README.md b/README.md index 1247885d5a..fc82a1a50b 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,7 @@ Destination fields describe details about the destination of a packet/event. Des | Field | Description | Level | Type | Example | |---|---|---|---|---| +| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field.
Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | extended | keyword | | | destination.ip | IP address of the destination.
Can be one or multiple IPv4 or IPv6 addresses. | core | ip | | | destination.port | Port of the destination. | core | long | | | destination.mac | MAC address of the destination. | core | keyword | | @@ -444,6 +445,7 @@ Source fields describe details about the source of a packet/event. Source fields | Field | Description | Level | Type | Example | |---|---|---|---|---| +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field.
Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | extended | keyword | | | source.ip | IP address of the source.
Can be one or multiple IPv4 or IPv6 addresses. | core | ip | | | source.port | Port of the source. | core | long | | | source.mac | MAC address of the source. | core | keyword | | diff --git a/fields.yml b/fields.yml index 3c1fa093ac..ae26b5402e 100644 --- a/fields.yml +++ b/fields.yml @@ -292,6 +292,17 @@ type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event destination addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip @@ -1243,6 +1254,17 @@ type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event source addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip diff --git a/schema.csv b/schema.csv index 99e5a80437..f55e7e341d 100644 --- a/schema.csv +++ b/schema.csv @@ -27,6 +27,7 @@ container.image.tag,keyword,extended, container.labels,object,extended, container.name,keyword,extended, container.runtime,keyword,extended,docker +destination.address,keyword,extended, destination.bytes,long,core,184 destination.domain,keyword,core, destination.ip,ip,core, @@ -140,6 +141,7 @@ service.name,keyword,core,elasticsearch-metrics service.state,keyword,core, service.type,keyword,core,elasticsearch service.version,keyword,core,3.2.4 +source.address,keyword,extended, source.bytes,long,core,184 source.domain,keyword,core, source.ip,ip,core, diff --git a/schemas/destination.yml b/schemas/destination.yml index a03e5a69bb..73c52d78f2 100644 --- a/schemas/destination.yml +++ b/schemas/destination.yml @@ -8,6 +8,17 @@ type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event destination addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip diff --git a/schemas/source.yml b/schemas/source.yml index a9e184c6bd..1efff6fe1c 100644 --- a/schemas/source.yml +++ b/schemas/source.yml @@ -8,6 +8,17 @@ type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event source addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip diff --git a/template.json b/template.json index 49361ff668..3b065979f9 100644 --- a/template.json +++ b/template.json @@ -148,6 +148,10 @@ }, "destination": { "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, "bytes": { "type": "long" }, @@ -665,6 +669,10 @@ }, "source": { "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, "bytes": { "type": "long" }, From 9b2c7a0cfb96646f69d21cf2d0a6b3c3e7f51505 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 6 Dec 2018 16:10:09 -0500 Subject: [PATCH 2/5] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed5dfd6de..534f4def7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ All notable changes to this project will be documented in this file based on the * Reintroduce a streamlined `user_agent` field set. #240 * Add `geo.name` for ad hoc location names. #248 * Add `event.timezone` to allow for proper interpretation of incomplete timestamps. #258 +* Add fields `source.address` and `destination.address`. #247 ### Improvements * Improved the definition of the file fields #196 From 7f65b2f8d2cbe6d82481888c027400e03b42660d Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 7 Dec 2018 12:26:29 -0500 Subject: [PATCH 3/5] Add the `address` field to client and server as well. --- README.md | 4 +++- fields.yml | 26 ++++++++++++++++++++++++-- schema.csv | 2 ++ schemas/client.yml | 13 ++++++++++++- schemas/server.yml | 11 +++++++++++ template.json | 8 ++++++++ 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fc82a1a50b..513e33f25a 100644 --- a/README.md +++ b/README.md @@ -107,11 +107,12 @@ Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it ## Client fields -A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. +A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. | Field | Description | Level | Type | Example | |---|---|---|---|---| +| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field.
Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | extended | keyword | | | client.ip | IP address of the client.
Can be one or multiple IPv4 or IPv6 addresses. | core | ip | | | client.port | Port of the client. | core | long | | | client.mac | MAC address of the client. | core | keyword | | @@ -415,6 +416,7 @@ A Server is defined as the responder in a network connection for events regardin | Field | Description | Level | Type | Example | |---|---|---|---|---| +| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field.
Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | extended | keyword | | | server.ip | IP address of the server.
Can be one or multiple IPv4 or IPv6 addresses. | core | ip | | | server.port | Port of the server. | core | long | | | server.mac | MAC address of the server. | core | keyword | | diff --git a/fields.yml b/fields.yml index ae26b5402e..1a4f8e5351 100644 --- a/fields.yml +++ b/fields.yml @@ -123,10 +123,21 @@ title: Client group: 2 description: > - A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. + A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event client addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip @@ -1258,7 +1269,7 @@ level: extended type: keyword description: > - Some event source addresses are defined ambiguously. The event will + Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. @@ -1395,6 +1406,17 @@ type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event source addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip diff --git a/schema.csv b/schema.csv index f55e7e341d..7f0264ce55 100644 --- a/schema.csv +++ b/schema.csv @@ -8,6 +8,7 @@ agent.id,keyword,core,8a4f500d agent.name,keyword,core,foo agent.type,keyword,core,filebeat agent.version,keyword,core,6.0.0-rc2 +client.address,keyword,extended, client.bytes,long,core,184 client.domain,keyword,core, client.ip,ip,core, @@ -129,6 +130,7 @@ process.thread.id,long,extended,4242 process.title,keyword,extended, process.working_directory,keyword,extended,/home/alice related.ip,ip,extended, +server.address,keyword,extended, server.bytes,long,core,184 server.domain,keyword,core, server.ip,ip,core, diff --git a/schemas/client.yml b/schemas/client.yml index e81a741d9d..a7e2bc2011 100644 --- a/schemas/client.yml +++ b/schemas/client.yml @@ -3,10 +3,21 @@ title: Client group: 2 description: > - A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. + A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event client addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip diff --git a/schemas/server.yml b/schemas/server.yml index 504b07300e..66246327b3 100644 --- a/schemas/server.yml +++ b/schemas/server.yml @@ -7,6 +7,17 @@ type: group fields: + - name: address + level: extended + type: keyword + description: > + Some event server addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which + one it is. + - name: ip level: core type: ip diff --git a/template.json b/template.json index 3b065979f9..77ebf8ff9c 100644 --- a/template.json +++ b/template.json @@ -49,6 +49,10 @@ }, "client": { "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, "bytes": { "type": "long" }, @@ -617,6 +621,10 @@ }, "server": { "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, "bytes": { "type": "long" }, From e4e568aa4769ed5ed35e85781e3fd05d697b8e21 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 7 Dec 2018 12:29:17 -0500 Subject: [PATCH 4/5] Fix 'conjuction' typo in client as well --- README.md | 2 +- fields.yml | 2 +- schemas/client.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 513e33f25a..29b178765b 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it ## Client fields -A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. +A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. | Field | Description | Level | Type | Example | diff --git a/fields.yml b/fields.yml index 1a4f8e5351..873e549c8c 100644 --- a/fields.yml +++ b/fields.yml @@ -123,7 +123,7 @@ title: Client group: 2 description: > - A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. + A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. type: group fields: diff --git a/schemas/client.yml b/schemas/client.yml index a7e2bc2011..5aa49f4924 100644 --- a/schemas/client.yml +++ b/schemas/client.yml @@ -3,7 +3,7 @@ title: Client group: 2 description: > - A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. + A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. type: group fields: From 8aa12a94449d30032b4bdc0694e399155d625697 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 11 Dec 2018 08:40:35 -0500 Subject: [PATCH 5/5] Update changelog to also list cli/srv --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 534f4def7a..e37b4468b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,8 @@ All notable changes to this project will be documented in this file based on the * Reintroduce a streamlined `user_agent` field set. #240 * Add `geo.name` for ad hoc location names. #248 * Add `event.timezone` to allow for proper interpretation of incomplete timestamps. #258 -* Add fields `source.address` and `destination.address`. #247 +* Add fields `source.address`, `destination.address`, `client.address`, and + `server.address`. #247 ### Improvements * Improved the definition of the file fields #196