Allow - for source IP for AWS S3 Access pipeline (#1333)

elastic · Jul 20, 2021 · 2d09c93 · 2d09c93
1 parent 8677ece
commit 2d09c93
Show file tree

Hide file tree

Showing 44 changed files with 161 additions and 75 deletions.
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.9.1"
+  changes:
+    - description: Fix `aws.s3access` pipeline when remote IP is a `-`
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1333
 - version: "0.9.0"
   changes:
     - description: Change default credential options to access keys

diff --git a/...s/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json b/...s/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json
@@ -25,7 +25,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.431910700Z",
+                "ingested": "2021-07-19T21:47:02.576945400Z",
                 "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}",
                 "provider": "iam.amazonaws.com",
                 "created": "2014-03-25T21:08:14.000Z",

diff --git a/...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json b/...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json
@@ -33,7 +33,7 @@
                 "ip": "123.145.67.89"
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.571172Z",
+                "ingested": "2021-07-19T21:47:02.627235200Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"[email protected]\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}",
                 "provider": "sts.amazonaws.com",
                 "created": "2019-10-02T22:12:29.000Z",

diff --git a/...aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json b/...aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json
@@ -24,7 +24,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.663769400Z",
+                "ingested": "2021-07-19T21:47:02.706667800Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:09:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"AccessDeniedException\",\"errorMessage\":\"An unknown error occurred\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE\",\"eventID\":\"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-09T00:09:33.000Z",
@@ -92,7 +92,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.663789500Z",
+                "ingested": "2021-07-19T21:47:02.706759900Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:03:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5c16-4eda-9724-EXAMPLE\",\"eventID\":\"EXAMPLE-35a7-4c25-9fc7-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-09T00:03:36.000Z",

diff --git a/...s/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json b/...s/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json
@@ -22,7 +22,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.712476Z",
+                "ingested": "2021-07-19T21:47:02.798942200Z",
                 "original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}",
                 "type": "info",
                 "kind": "event"

diff --git a/...s/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json b/...s/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json
@@ -24,7 +24,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.732317900Z",
+                "ingested": "2021-07-19T21:47:02.827257900Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JohnDoe\",\"accountId\":\"111122223333\",\"userName\":\"JohnDoe\"},\"eventTime\":\"2014-07-16T15:49:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.110\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Success\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/s3/\",\"MFAUsed\":\"No\"},\"eventID\":\"3fcfb182-98f8-4744-bd45-10aEXAMPLE\"}",
                 "provider": "signin.amazonaws.com",
                 "created": "2014-07-16T15:49:27.000Z",
@@ -109,7 +109,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.732335700Z",
+                "ingested": "2021-07-19T21:47:02.827288300Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JaneDoe\",\"accountId\":\"111122223333\",\"userName\":\"JaneDoe\"},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
                 "provider": "signin.amazonaws.com",
                 "created": "2014-07-08T17:35:27.000Z",
@@ -187,7 +187,7 @@
                 "ip": "192.0.2.100"
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.732341900Z",
+                "ingested": "2021-07-19T21:47:02.827299600Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
                 "provider": "signin.amazonaws.com",
                 "created": "2014-07-08T17:35:27.000Z",

diff --git a/...s/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json b/...s/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json
@@ -25,7 +25,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.822409700Z",
+                "ingested": "2021-07-19T21:47:02.960333Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:43:06Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"accessKey\":{\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"status\":\"Active\",\"userName\":\"Bob\",\"createDate\":\"Jan 8, 2020 8:43:06 PM\"}},\"requestID\":\"EXAMPLE-823a-48dc-8fa9-EXAMPLE\",\"eventID\":\"EXAMPLE-3cab-40f8-938b-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-08T20:43:06.000Z",

diff --git a/...es/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json b/...es/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json
@@ -24,7 +24,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.853338900Z",
+                "ingested": "2021-07-19T21:47:03.006393400Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T01:48:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":{\"group\":{\"createDate\":\"Jan 9, 2020 1:48:44 AM\",\"path\":\"/\",\"arn\":\"arn:aws:iam::0123456789012:group/TEST-GROUP\",\"groupName\":\"TEST-GROUP\",\"groupId\":\"EXAMPLE_ID\"}},\"requestID\":\"EXAMPLE-769d-4a61-b731-EXAMPLE\",\"eventID\":\"EXAMPLE-37ec-425a-a7ef-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-09T01:48:44.000Z",
@@ -113,7 +113,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-06-04T09:43:14.853361900Z",
+                "ingested": "2021-07-19T21:47:03.006400200Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:22:03Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c8ae-44dc-8114-EXAMPLE\",\"eventID\":\"EXAMPLE-09c6-4745-af70-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-09T02:22:03.000Z",