From df73e556e934936a56358c7c394f7ad1bd785979 Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Mon, 18 Oct 2021 19:51:49 +0300 Subject: [PATCH 1/4] poc of non supported regions errors ignoring --- client/helpers.go | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/client/helpers.go b/client/helpers.go index aa818f667..ddfd65d5a 100644 --- a/client/helpers.go +++ b/client/helpers.go @@ -1,9 +1,14 @@ package client import ( + "encoding/json" "errors" "fmt" + "io/ioutil" + "net/http" "regexp" + "strings" + "sync" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/smithy-go" @@ -12,7 +17,81 @@ import ( //log-group:([a-zA-Z0-9/]+): var GroupNameRegex = regexp.MustCompile("arn:aws:logs:[a-z0-9-]+:[0-9]+:log-group:([a-zA-Z0-9-/]+):") +type SupportedServicesData struct { + Prices []struct { + Attributes struct { + Region string `json:"aws:region"` + Service string `json:"aws:serviceName"` + } `json:"attributes"` + Id string `json:"id"` + } `json:"prices"` +} + +var supportedServices *SupportedServicesData + +func downloadSupportedResourcesForRegions() (*SupportedServicesData, error) { + // Get the data + req, err := http.NewRequest(http.MethodGet, "https://api.regional-table.region-services.aws.a2z.com/index.json", nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get aws supported resources for region, status code: %d", resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var data SupportedServicesData + err = json.Unmarshal(body, &data) + if err != nil { + return nil, err + } + + for i, item := range data.Prices { + // replacing the name because name from api does not always fit + parts := strings.Split(item.Id, ":") + data.Prices[i].Attributes.Service = parts[0] + } + + return &data, nil +} + +var once sync.Once + +func checkUnsupportedResourceForRegionError(err error) bool { + once.Do(func() { + supportedServices, _ = downloadSupportedResourcesForRegions() + }) + errText := err.Error() + + if supportedServices != nil && strings.Contains(errText, "no such host") { + var ae *smithy.OperationError + if errors.As(err, &ae) { + for _, p := range supportedServices.Prices { + pattern := fmt.Sprintf("lookup %s.+%s\\.amazonaws\\.com", p.Attributes.Service, p.Attributes.Region) + // match means that error is related to a supported service + if match, _ := regexp.MatchString(pattern, errText); match { + return false + } + } + } + } + return true +} + func IgnoreAccessDeniedServiceDisabled(err error) bool { + if checkUnsupportedResourceForRegionError(err) { + return true + } var ae smithy.APIError if errors.As(err, &ae) { switch ae.ErrorCode() { From d2da390ea2719dfa7865ffccd84b72e4e661bce3 Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Tue, 19 Oct 2021 14:59:01 +0300 Subject: [PATCH 2/4] optimized error handling --- client/helpers.go | 70 +++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/client/helpers.go b/client/helpers.go index ddfd65d5a..ad6b00497 100644 --- a/client/helpers.go +++ b/client/helpers.go @@ -4,7 +4,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "net" "net/http" "regexp" "strings" @@ -14,6 +14,8 @@ import ( "github.com/aws/smithy-go" ) +const supportedServicesLink = "https://api.regional-table.region-services.aws.a2z.com/index.json" + //log-group:([a-zA-Z0-9/]+): var GroupNameRegex = regexp.MustCompile("arn:aws:logs:[a-z0-9-]+:[0-9]+:log-group:([a-zA-Z0-9-/]+):") @@ -27,11 +29,20 @@ type SupportedServicesData struct { } `json:"prices"` } -var supportedServices *SupportedServicesData +// supportedServices map of the supported service-regions +var supportedServices map[string]map[string]struct{} +var getSupportedServices sync.Once + +// apiErrorServiceNames stores api subdomains and service names for error decoding +var apiErrorServiceNames = map[string]string{ + "mq": "Amazon MQ", + "cognito-identity": "Amazon Cognito", + "cognito-idp": "Amazon Cognito", + "ec2": "Amazon Elastic Compute Cloud (EC2)", +} -func downloadSupportedResourcesForRegions() (*SupportedServicesData, error) { - // Get the data - req, err := http.NewRequest(http.MethodGet, "https://api.regional-table.region-services.aws.a2z.com/index.json", nil) +func downloadSupportedResourcesForRegions() (map[string]map[string]struct{}, error) { + req, err := http.NewRequest(http.MethodGet, supportedServicesLink, nil) if err != nil { return nil, err } @@ -45,51 +56,40 @@ func downloadSupportedResourcesForRegions() (*SupportedServicesData, error) { return nil, fmt.Errorf("failed to get aws supported resources for region, status code: %d", resp.StatusCode) } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - var data SupportedServicesData - err = json.Unmarshal(body, &data) - if err != nil { + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { return nil, err } - for i, item := range data.Prices { - // replacing the name because name from api does not always fit - parts := strings.Split(item.Id, ":") - data.Prices[i].Attributes.Service = parts[0] + m := make(map[string]map[string]struct{}) + for _, p := range data.Prices { + if _, ok := m[p.Attributes.Service]; !ok { + m[p.Attributes.Service] = make(map[string]struct{}) + } + m[p.Attributes.Service][p.Attributes.Region] = struct{}{} } - return &data, nil + return m, nil } -var once sync.Once - -func checkUnsupportedResourceForRegionError(err error) bool { - once.Do(func() { +func ignoreUnsupportedResourceForRegionError(err error) bool { + getSupportedServices.Do(func() { supportedServices, _ = downloadSupportedResourcesForRegions() }) - errText := err.Error() - - if supportedServices != nil && strings.Contains(errText, "no such host") { - var ae *smithy.OperationError - if errors.As(err, &ae) { - for _, p := range supportedServices.Prices { - pattern := fmt.Sprintf("lookup %s.+%s\\.amazonaws\\.com", p.Attributes.Service, p.Attributes.Region) - // match means that error is related to a supported service - if match, _ := regexp.MatchString(pattern, errText); match { - return false - } - } - } + var dnsErr *net.DNSError + if supportedServices != nil && errors.As(err, &dnsErr) && dnsErr.IsNotFound { + parts := strings.Split(dnsErr.Name, ".") + apiService := apiErrorServiceNames[parts[0]] + region := parts[1] + _, ok := supportedServices[apiService][region] + // if service-region combination is in the map than service is supported and error should not be ignored + return ok } return true } func IgnoreAccessDeniedServiceDisabled(err error) bool { - if checkUnsupportedResourceForRegionError(err) { + if ignoreUnsupportedResourceForRegionError(err) { return true } var ae smithy.APIError From c772e71346deeabef7d7f59700dc6c00b079065f Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Tue, 19 Oct 2021 21:01:33 +0300 Subject: [PATCH 3/4] added more error checks --- client/helpers.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/client/helpers.go b/client/helpers.go index ad6b00497..01f8302f9 100644 --- a/client/helpers.go +++ b/client/helpers.go @@ -79,9 +79,17 @@ func ignoreUnsupportedResourceForRegionError(err error) bool { var dnsErr *net.DNSError if supportedServices != nil && errors.As(err, &dnsErr) && dnsErr.IsNotFound { parts := strings.Split(dnsErr.Name, ".") - apiService := apiErrorServiceNames[parts[0]] + if len(parts) < 2 { + // usual aws domain has more than 2 parts + return false + } + apiService, ok := apiErrorServiceNames[parts[0]] + if !ok { + return false + } region := parts[1] - _, ok := supportedServices[apiService][region] + + _, ok = supportedServices[apiService][region] // if service-region combination is in the map than service is supported and error should not be ignored return ok } From b044371cb18cdb86e323fa488358cae60dc4f485 Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Thu, 21 Oct 2021 17:29:34 +0300 Subject: [PATCH 4/4] functions descriptions added --- client/helpers.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/helpers.go b/client/helpers.go index 01f8302f9..13060a712 100644 --- a/client/helpers.go +++ b/client/helpers.go @@ -14,8 +14,6 @@ import ( "github.com/aws/smithy-go" ) -const supportedServicesLink = "https://api.regional-table.region-services.aws.a2z.com/index.json" - //log-group:([a-zA-Z0-9/]+): var GroupNameRegex = regexp.MustCompile("arn:aws:logs:[a-z0-9-]+:[0-9]+:log-group:([a-zA-Z0-9-/]+):") @@ -41,6 +39,9 @@ var apiErrorServiceNames = map[string]string{ "ec2": "Amazon Elastic Compute Cloud (EC2)", } +const supportedServicesLink = "https://api.regional-table.region-services.aws.a2z.com/index.json" + +// downloadSupportedResourcesForRegions gets the data about AWS services and regions they are available in func downloadSupportedResourcesForRegions() (map[string]map[string]struct{}, error) { req, err := http.NewRequest(http.MethodGet, supportedServicesLink, nil) if err != nil { @@ -72,6 +73,7 @@ func downloadSupportedResourcesForRegions() (map[string]map[string]struct{}, err return m, nil } +// ignoreUnsupportedResourceForRegionError returns true request was sent to a service that does not exist in specified region func ignoreUnsupportedResourceForRegionError(err error) bool { getSupportedServices.Do(func() { supportedServices, _ = downloadSupportedResourcesForRegions()