From c8119dfe7e83ffd39fc30780795a4bfbb962780d Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 16 Jul 2021 16:01:49 +0000 Subject: [PATCH 1/2] #26913: Allow - for source IP for AWS S3 Access pipeline --- .../module/aws/s3access/ingest/pipeline.yml | 2 +- .../aws/s3access/test/s3_server_access.log | 1 + .../test/s3_server_access.log-expected.json | 54 +++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index 8cfaa7109c0..006fbc0934c 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -15,7 +15,7 @@ processors: patterns: - >- %{BASE16NUM:aws.s3access.bucket_owner} %{HOSTNAME:aws.s3access.bucket} \[%{HTTPDATE:_temp_.s3access_time}\] - %{IP:aws.s3access.remote_ip} (?:-|%{S3REQUESTER:aws.s3access.requester}) %{S3REQUESTID:aws.s3access.request_id} + (?:-|%{IP:aws.s3access.remote_ip}) (?:-|%{S3REQUESTER:aws.s3access.requester}) %{S3REQUESTID:aws.s3access.request_id} %{S3OPERATION:aws.s3access.operation} (?:-|%{S3KEY:aws.s3access.key}) (?:-|\"%{DATA:aws.s3access.request_uri}\") %{NUMBER:aws.s3access.http_status:long} (?:-|%{WORD:aws.s3access.error_code}) (?:-|%{NUMBER:aws.s3access.bytes_sent:long}) (?:-|%{NUMBER:aws.s3access.object_size:long}) (?:-|%{NUMBER:aws.s3access.total_time:long}) (?:-|%{NUMBER:aws.s3access.turn_around_time:long}) diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log index 72b45f88ef2..e56b8a34ed9 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log @@ -4,3 +4,4 @@ 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b flow-log-test [14/Jul/2021:18:57:31 +0000] - svc:delivery.logs.amazonaws.com MVGXZXEVN3IG9S24 REST.PUT.OBJECT AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz "PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1" 200 - - 773 103 13 "-" "-" - 02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader flow-log-test.s3.us-gov-west-1.amazonaws.com TLSv1.2 - diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json index 37345ff30fc..80676a1b6cb 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json @@ -417,5 +417,59 @@ "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", "tls.version_protocol": "tls" + }, + { + "@timestamp": "2021-07-14T18:57:31.000Z", + "aws.s3access.authentication_type": "AuthHeader", + "aws.s3access.bucket": "flow-log-test", + "aws.s3access.bucket_owner": "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b", + "aws.s3access.cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.s3access.host_header": "flow-log-test.s3.us-gov-west-1.amazonaws.com", + "aws.s3access.host_id": "02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4=", + "aws.s3access.http_status": 200, + "aws.s3access.key": "AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz", + "aws.s3access.object_size": 773, + "aws.s3access.operation": "REST.PUT.OBJECT", + "aws.s3access.request_id": "MVGXZXEVN3IG9S24", + "aws.s3access.request_uri": "PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1", + "aws.s3access.requester": "svc:delivery.logs.amazonaws.com", + "aws.s3access.signature_version": "SigV4", + "aws.s3access.tls_version": "TLSv1.2", + "aws.s3access.total_time": 103, + "aws.s3access.turn_around_time": 13, + "client.user.id": "svc:delivery.logs.amazonaws.com", + "cloud.provider": "aws", + "cloud.region": "us-gov-west-1", + "event.action": "REST.PUT.OBJECT", + "event.category": "web", + "event.dataset": "aws.s3access", + "event.duration": 103000000, + "event.id": "MVGXZXEVN3IG9S24", + "event.kind": "event", + "event.module": "aws", + "event.original": "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b flow-log-test [14/Jul/2021:18:57:31 +0000] - svc:delivery.logs.amazonaws.com MVGXZXEVN3IG9S24 REST.PUT.OBJECT AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz \"PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1\" 200 - - 773 103 13 \"-\" \"-\" - 02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader flow-log-test.s3.us-gov-west-1.amazonaws.com TLSv1.2 -", + "event.outcome": "success", + "event.type": [ + "access" + ], + "fileset.name": "s3access", + "http.request.method": "PUT", + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 3700, + "related.user": [ + "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b" + ], + "service.type": "aws", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "url.extension": "gz", + "url.original": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz", + "url.path": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz" } ] \ No newline at end of file From 20ae3261b087014dcdd340b15fd94759f0387e89 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 16 Jul 2021 16:07:05 +0000 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index fff3da1d38f..77688830067 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -402,6 +402,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Suricata metadata fields breaking visualizations, moved out of flattened datatype. {pull}26710[26710] - Fix `httpjson` template data key for `url.params`. {pull}26848[26848] - Cisco asa/ftd: Fix reversed usage of observer ingress and egress interfaces. {pull}26265[26265] +- Fix `aws.s3access` pipeline when remote IP is a `-`. {issue}26913[26913] {pull}26940[26940] *Heartbeat*