From 5069e6495ef092df013a61071a4330a61651ca70 Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Wed, 8 Jun 2022 15:08:39 +0300 Subject: [PATCH 1/3] fix: Added not found checks in athena workgroups --- resources/services/athena/work_groups.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/resources/services/athena/work_groups.go b/resources/services/athena/work_groups.go index 9f8fbf02d..b25c45728 100644 --- a/resources/services/athena/work_groups.go +++ b/resources/services/athena/work_groups.go @@ -428,7 +428,7 @@ func fetchAthenaWorkGroups(ctx context.Context, meta schema.ClientMeta, parent * func(summary types.WorkGroupSummary) { errs.Go(func() error { defer sem.Release(1) - return fetchWorkGroup(ctx, res, svc, c.Region, summary) + return fetchWorkGroup(ctx, res, c, c.Region, summary) }) }(d) } @@ -491,6 +491,9 @@ func fetchAthenaWorkGroupPreparedStatements(ctx context.Context, meta schema.Cli options.Region = c.Region }) if err != nil { + if c.IsNotFoundError(err) { + continue + } return diag.WrapError(err) } res <- *dc.PreparedStatement @@ -517,12 +520,14 @@ func fetchAthenaWorkGroupQueryExecutions(ctx context.Context, meta schema.Client } for _, d := range response.QueryExecutionIds { dc, err := svc.GetQueryExecution(ctx, &athena.GetQueryExecutionInput{ - QueryExecutionId: aws.String(d), }, func(options *athena.Options) { options.Region = c.Region }) if err != nil { + if c.IsNotFoundError(err) { + continue + } return diag.WrapError(err) } res <- *dc.QueryExecution @@ -554,6 +559,9 @@ func fetchAthenaWorkGroupNamedQueries(ctx context.Context, meta schema.ClientMet options.Region = c.Region }) if err != nil { + if c.IsNotFoundError(err) { + continue + } return diag.WrapError(err) } res <- *dc.NamedQuery @@ -571,13 +579,17 @@ func fetchAthenaWorkGroupNamedQueries(ctx context.Context, meta schema.ClientMet // User Defined Helpers // ==================================================================================================================== -func fetchWorkGroup(ctx context.Context, res chan<- interface{}, svc client.AthenaClient, region string, groupSummary types.WorkGroupSummary) error { +func fetchWorkGroup(ctx context.Context, res chan<- interface{}, c *client.Client, region string, groupSummary types.WorkGroupSummary) error { + svc := c.Services().Athena dc, err := svc.GetWorkGroup(ctx, &athena.GetWorkGroupInput{ WorkGroup: groupSummary.Name, }, func(options *athena.Options) { options.Region = region }) if err != nil { + if c.IsNotFoundError(err) { + return nil + } return diag.WrapError(err) } res <- *dc.WorkGroup From d0022184953f67bc4d0a840f22032494fe2e3911 Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Wed, 8 Jun 2022 15:26:16 +0300 Subject: [PATCH 2/3] added data catalogs --- resources/services/athena/data_catalogs.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/resources/services/athena/data_catalogs.go b/resources/services/athena/data_catalogs.go index 0d5537383..b422717a3 100644 --- a/resources/services/athena/data_catalogs.go +++ b/resources/services/athena/data_catalogs.go @@ -234,7 +234,7 @@ func fetchAthenaDataCatalogs(ctx context.Context, meta schema.ClientMeta, parent func(summary types.DataCatalogSummary) { errs.Go(func() error { defer sem.Release(1) - return fetchDataCatalog(ctx, res, svc, c.Region, summary) + return fetchDataCatalog(ctx, res, c, c.Region, summary) }) }(d) } @@ -335,7 +335,8 @@ func fetchAthenaDataCatalogDatabaseTablePartitionKeys(ctx context.Context, meta // User Defined Helpers // ==================================================================================================================== -func fetchDataCatalog(ctx context.Context, res chan<- interface{}, svc client.AthenaClient, region string, catalogSummary types.DataCatalogSummary) error { +func fetchDataCatalog(ctx context.Context, res chan<- interface{}, c *client.Client, region string, catalogSummary types.DataCatalogSummary) error { + svc := c.Services().Athena dc, err := svc.GetDataCatalog(ctx, &athena.GetDataCatalogInput{ Name: catalogSummary.CatalogName, }, func(options *athena.Options) { @@ -348,6 +349,9 @@ func fetchDataCatalog(ctx context.Context, res chan<- interface{}, svc client.At res <- types.DataCatalog{Name: catalogSummary.CatalogName, Type: catalogSummary.Type} return nil } + if c.IsNotFoundError(err) { + return nil + } return diag.WrapError(err) } res <- *dc.DataCatalog From a87a25c8c3ed44ae30417243ab6a0e3915a97845 Mon Sep 17 00:00:00 2001 From: Andrii Romanenko Date: Wed, 8 Jun 2022 17:23:02 +0300 Subject: [PATCH 3/3] removed region argument, added error handler --- resources/services/athena/data_catalogs.go | 6 +++--- resources/services/athena/work_groups.go | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/resources/services/athena/data_catalogs.go b/resources/services/athena/data_catalogs.go index b422717a3..7d7b5b893 100644 --- a/resources/services/athena/data_catalogs.go +++ b/resources/services/athena/data_catalogs.go @@ -234,7 +234,7 @@ func fetchAthenaDataCatalogs(ctx context.Context, meta schema.ClientMeta, parent func(summary types.DataCatalogSummary) { errs.Go(func() error { defer sem.Release(1) - return fetchDataCatalog(ctx, res, c, c.Region, summary) + return fetchDataCatalog(ctx, res, c, summary) }) }(d) } @@ -335,12 +335,12 @@ func fetchAthenaDataCatalogDatabaseTablePartitionKeys(ctx context.Context, meta // User Defined Helpers // ==================================================================================================================== -func fetchDataCatalog(ctx context.Context, res chan<- interface{}, c *client.Client, region string, catalogSummary types.DataCatalogSummary) error { +func fetchDataCatalog(ctx context.Context, res chan<- interface{}, c *client.Client, catalogSummary types.DataCatalogSummary) error { svc := c.Services().Athena dc, err := svc.GetDataCatalog(ctx, &athena.GetDataCatalogInput{ Name: catalogSummary.CatalogName, }, func(options *athena.Options) { - options.Region = region + options.Region = c.Region }) if err != nil { // retrieving of default data catalog (AwsDataCatalog) returns "not found error" but it exists and its diff --git a/resources/services/athena/work_groups.go b/resources/services/athena/work_groups.go index b25c45728..d0ee126ba 100644 --- a/resources/services/athena/work_groups.go +++ b/resources/services/athena/work_groups.go @@ -2,10 +2,13 @@ package athena import ( "context" + "errors" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" + "github.com/aws/smithy-go" "github.com/cloudquery/cq-provider-aws/client" "github.com/cloudquery/cq-provider-sdk/provider/diag" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -428,7 +431,7 @@ func fetchAthenaWorkGroups(ctx context.Context, meta schema.ClientMeta, parent * func(summary types.WorkGroupSummary) { errs.Go(func() error { defer sem.Release(1) - return fetchWorkGroup(ctx, res, c, c.Region, summary) + return fetchWorkGroup(ctx, res, c, summary) }) }(d) } @@ -525,7 +528,7 @@ func fetchAthenaWorkGroupQueryExecutions(ctx context.Context, meta schema.Client options.Region = c.Region }) if err != nil { - if c.IsNotFoundError(err) { + if c.IsNotFoundError(err) || isQueryExecutionNotFound(err) { continue } return diag.WrapError(err) @@ -579,12 +582,12 @@ func fetchAthenaWorkGroupNamedQueries(ctx context.Context, meta schema.ClientMet // User Defined Helpers // ==================================================================================================================== -func fetchWorkGroup(ctx context.Context, res chan<- interface{}, c *client.Client, region string, groupSummary types.WorkGroupSummary) error { +func fetchWorkGroup(ctx context.Context, res chan<- interface{}, c *client.Client, groupSummary types.WorkGroupSummary) error { svc := c.Services().Athena dc, err := svc.GetWorkGroup(ctx, &athena.GetWorkGroupInput{ WorkGroup: groupSummary.Name, }, func(options *athena.Options) { - options.Region = region + options.Region = c.Region }) if err != nil { if c.IsNotFoundError(err) { @@ -599,3 +602,11 @@ func fetchWorkGroup(ctx context.Context, res chan<- interface{}, c *client.Clien func createWorkGroupArn(cl *client.Client, groupName string) string { return cl.ARN(client.Athena, "workgroup", groupName) } + +func isQueryExecutionNotFound(err error) bool { + var ae smithy.APIError + if !errors.As(err, &ae) { + return false + } + return ae.ErrorCode() == "InvalidRequestException" && strings.Contains(ae.ErrorMessage(), "was not found") +}