From e46371be7b6522d42465db994cf97f49475da2f0 Mon Sep 17 00:00:00 2001 From: Timur Irmatov Date: Mon, 20 Jun 2022 14:08:53 +0500 Subject: [PATCH] fix: Redact separate error messages separately (#1071) This splits single lookupRegex into two: * one matches lookup failures * second matches read errors --- client/errors.go | 20 ++++++++++++++------ client/errors_test.go | 6 +++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/client/errors.go b/client/errors.go index b3204a8ac..602ea279c 100644 --- a/client/errors.go +++ b/client/errors.go @@ -15,11 +15,18 @@ import ( const ssoInvalidOrExpired = "failed to refresh cached credentials, the SSO session has expired or is invalid" var ( - requestIdRegex = regexp.MustCompile(`\s([Rr]equest[ _]{0,1}(ID|Id|id):)\s[A-Za-z0-9-]+`) - hostIdRegex = regexp.MustCompile(`\sHostID: [A-Za-z0-9+/_=-]+`) - arnIdRegex = regexp.MustCompile(`(\s)(arn:aws[A-Za-z0-9-]*:)[^ \.\(\)\[\]\{\}\;\,]+(\s?)`) - urlRegex = regexp.MustCompile(`([\s"])http(s?):\/\/[a-z0-9_\-\./]+([":\s]?)`) - lookupRegex = regexp.MustCompile(`(\slookup\s)[-A-Za-z0-9\.]+\son\s([0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}:[0-9]{1,5})(:.+?)([0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}:[0-9]{1,5})->([0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}:[0-9]{1,5})(:.*)`) + requestIdRegex = regexp.MustCompile(`\s([Rr]equest[ _]{0,1}(ID|Id|id):)\s[A-Za-z0-9-]+`) + hostIdRegex = regexp.MustCompile(`\sHostID: [A-Za-z0-9+/_=-]+`) + arnIdRegex = regexp.MustCompile(`(\s)(arn:aws[A-Za-z0-9-]*:)[^ \.\(\)\[\]\{\}\;\,]+(\s?)`) + urlRegex = regexp.MustCompile(`([\s"])http(s?):\/\/[a-z0-9_\-\./]+([":\s]?)`) + lookupRegex = regexp.MustCompile( + `\blookup\s[-A-Za-z0-9\.]+\s` + // " lookup host.name " + `on\s\S+:\d+`, // "on 123.123.123.123:53" + ) + readXonYRegex = regexp.MustCompile( + `\bread\s(udp|tcp)\s` + // "read udp " + `\S+:\d+->\S+:\d+`, // "192.168.1.2:5353->192.168.1.1:53" + ) dialRegex = regexp.MustCompile(`(\sdial\s)(tcp|udp)(\s)([0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}:[0-9]{1,5})(:.+?)`) encAuthRegex = regexp.MustCompile(`(\s)(Encoded authorization failure message:)\s[A-Za-z0-9_-]+`) userRegex = regexp.MustCompile(`(\s)(is not authorized to perform: .+ on resource:\s)(user)\s.+`) @@ -235,7 +242,8 @@ func removePII(aa []string, msg string) string { msg = hostIdRegex.ReplaceAllString(msg, " HostID: xxxx") msg = arnIdRegex.ReplaceAllString(msg, "${1}${2}xxxx${3}") msg = urlRegex.ReplaceAllString(msg, "${1}http${2}://xxxx${3}") - msg = lookupRegex.ReplaceAllString(msg, "${1}xxxx${3}xxxx->xxxx${6}") + msg = lookupRegex.ReplaceAllString(msg, "lookup xxxx on xxxx:xx") + msg = readXonYRegex.ReplaceAllString(msg, "read $1 xxxx:xx->xxxx:xx") msg = dialRegex.ReplaceAllString(msg, "${1}${2}${3}xxxx${5}") msg = encAuthRegex.ReplaceAllString(msg, "${1}${2} xxxx") msg = userRegex.ReplaceAllString(msg, "${1}${2}${3} xxxx") diff --git a/client/errors_test.go b/client/errors_test.go index 861a6ea7b..b652d1a21 100644 --- a/client/errors_test.go +++ b/client/errors_test.go @@ -33,7 +33,7 @@ func TestRemovePII(t *testing.T) { }, { "operation error CloudWatch Logs: DescribeMetricFilters, exceeded maximum number of attempts, 10, https response error StatusCode: 0, RequestID: , request send failed, Post \"https://logs.eu-central-1.amazonaws.com/\": dial tcp: lookup logs.eu-central-1.amazonaws.com on 192.168.1.1:53: read udp 192.168.1.2:5353->192.168.1.1:53: i/o timeout", - "operation error CloudWatch Logs: DescribeMetricFilters, exceeded maximum number of attempts, 10, https response error StatusCode: 0, RequestID: , request send failed, Post \"https://xxxx\": dial tcp: lookup xxxx: read udp xxxx->xxxx: i/o timeout", + "operation error CloudWatch Logs: DescribeMetricFilters, exceeded maximum number of attempts, 10, https response error StatusCode: 0, RequestID: , request send failed, Post \"https://xxxx\": dial tcp: lookup xxxx on xxxx:xx: read udp xxxx:xx->xxxx:xx: i/o timeout", }, { "EC2: DescribeImageAttribute - You are not authorized to perform this operation. Encoded authorization failure message: SOMEENCODEDMESSAGEWITHNUMBERS1234567ANDDASHANDUNDERSCORES-ABCDE_123123123_EXAMPLEMESSAGE", @@ -55,6 +55,10 @@ func TestRemovePII(t *testing.T) { "ResourceType name not found - Could not find example request type named 'resource-dev-1111'", "ResourceType name not found - Could not find example request type named 'xxxx'", }, + { + `qldb.ledgers: failed to resolve table "aws_qldb_ledgers": error at github.com/cloudquery/cq-provider-aws/resources/services/qldb.fetchQldbLedgers[ledgers.go:264] operation error QLDB: ListLedgers, exceeded maximum number of attempts, 10, https response error StatusCode: 0, RequestID: , request send failed, Get "https://qldb.ap-southeast-1.amazonaws.com/ledgers": dial tcp: lookup qldb.ap-southeast-1.amazonaws.com on 172.20.0.10:53: no such host`, + `qldb.ledgers: failed to resolve table "aws_qldb_ledgers": error at github.com/cloudquery/cq-provider-aws/resources/services/qldb.fetchQldbLedgers[ledgers.go:264] operation error QLDB: ListLedgers, exceeded maximum number of attempts, 10, https response error StatusCode: 0, RequestID: , request send failed, Get "https://xxxx": dial tcp: lookup xxxx on xxxx:xx: no such host`, + }, } for i, tc := range cases { res := removePII([]string{"123456789"}, tc.Input)