From 2e24cc26955028644fe4f5ac68f354f09dd3e153 Mon Sep 17 00:00:00 2001 From: bbernays Date: Thu, 9 Dec 2021 12:03:03 -0500 Subject: [PATCH 1/8] Update execution.go --- provider/schema/execution.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/provider/schema/execution.go b/provider/schema/execution.go index 40a23251..007bf1a6 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "runtime/debug" + "strings" "sync/atomic" "time" @@ -197,6 +198,9 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, err := e.Table.Resolver(ctx, client, parent, res) if err != nil && e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) + if strings.Contains(err.Error(), ": socket: too many open files") { + client.Logger().Warn("try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") + } // add partial fetch error, this will be passed in diagnostics, although it was ignored _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") return From 45408da4e9188cd371fbd6c02de951a5d3f69135 Mon Sep 17 00:00:00 2001 From: bbernays Date: Sun, 12 Dec 2021 15:08:44 -0500 Subject: [PATCH 2/8] Respond to comments --- provider/provider.go | 6 ++++++ provider/schema/execution.go | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/provider/provider.go b/provider/provider.go index a7a0ab3d..f806a691 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -5,6 +5,7 @@ import ( "embed" "errors" "fmt" + "strings" "sync" "sync/atomic" @@ -259,6 +260,11 @@ func (p *Provider) collectExecutionDiagnostics(client schema.ClientMeta, exec sc diagnostics = append(diagnostics, dd...) continue } + + if strings.Contains(e.Error(), ": socket: too many open files") { + diagnostics = append(diagnostics, diag.FromError(e.Err, diag.WARNING, diag.THROTTLE, exec.ResourceName, e.Error(), "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters")) + continue + } // if error wasn't classified by provider mark it as error diagnostics = append(diagnostics, diag.FromError(e.Err, diag.ERROR, diag.RESOLVING, exec.ResourceName, e.Error(), "")) } diff --git a/provider/schema/execution.go b/provider/schema/execution.go index 007bf1a6..0d037131 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -196,14 +196,21 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, close(res) }() err := e.Table.Resolver(ctx, client, parent, res) - if err != nil && e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { - client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) + if err != nil { + if e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { + client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) + // add partial fetch error, this will be passed in diagnostics, although it was ignored + _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") + return + } + + // Not sure if this should be before we check for an IgnoredError as it is a unique error that we know about locally if strings.Contains(err.Error(), ": socket: too many open files") { client.Logger().Warn("try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") + _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") + return + } - // add partial fetch error, this will be passed in diagnostics, although it was ignored - _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") - return } resolverErr = err }() From e36f7ae4b427951c8a4ddf15c8278f9941c77f45 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 13 Dec 2021 09:07:26 -0500 Subject: [PATCH 3/8] simplify --- provider/provider.go | 5 ----- provider/schema/execution.go | 24 +++++++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/provider/provider.go b/provider/provider.go index f806a691..5bc6dab8 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -5,7 +5,6 @@ import ( "embed" "errors" "fmt" - "strings" "sync" "sync/atomic" @@ -261,10 +260,6 @@ func (p *Provider) collectExecutionDiagnostics(client schema.ClientMeta, exec sc continue } - if strings.Contains(e.Error(), ": socket: too many open files") { - diagnostics = append(diagnostics, diag.FromError(e.Err, diag.WARNING, diag.THROTTLE, exec.ResourceName, e.Error(), "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters")) - continue - } // if error wasn't classified by provider mark it as error diagnostics = append(diagnostics, diag.FromError(e.Err, diag.ERROR, diag.RESOLVING, exec.ResourceName, e.Error(), "")) } diff --git a/provider/schema/execution.go b/provider/schema/execution.go index 0d037131..c2b81ae0 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" "github.com/modern-go/reflect2" _ "github.com/doug-martin/goqu/v9/dialect/postgres" @@ -196,21 +197,11 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, close(res) }() err := e.Table.Resolver(ctx, client, parent, res) - if err != nil { - if e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { - client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) - // add partial fetch error, this will be passed in diagnostics, although it was ignored - _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") - return - } - - // Not sure if this should be before we check for an IgnoredError as it is a unique error that we know about locally - if strings.Contains(err.Error(), ": socket: too many open files") { - client.Logger().Warn("try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") - _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") - return - - } + if err != nil && e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { + client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) + // add partial fetch error, this will be passed in diagnostics, although it was ignored + _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") + return } resolverErr = err }() @@ -229,6 +220,9 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, // check if channel iteration stopped because of resolver failure if resolverErr != nil { client.Logger().Error("received resolve resources error", "table", e.Table.Name, "error", resolverErr) + if strings.Contains(resolverErr.Error(), ": socket: too many open files") { + return 0, diag.FromError(resolverErr, diag.WARNING, diag.THROTTLE, e.ResourceName, resolverErr.Error(), "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") + } return 0, e.checkPartialFetchError(resolverErr, nil, "table resolve error") } // Print only parent resources From 33d07e18a87f4f373da471b0ceeaa8949c61f963 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 13 Dec 2021 09:08:01 -0500 Subject: [PATCH 4/8] Update provider.go --- provider/provider.go | 1 - 1 file changed, 1 deletion(-) diff --git a/provider/provider.go b/provider/provider.go index 5bc6dab8..a7a0ab3d 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -259,7 +259,6 @@ func (p *Provider) collectExecutionDiagnostics(client schema.ClientMeta, exec sc diagnostics = append(diagnostics, dd...) continue } - // if error wasn't classified by provider mark it as error diagnostics = append(diagnostics, diag.FromError(e.Err, diag.ERROR, diag.RESOLVING, exec.ResourceName, e.Error(), "")) } From 628ec2bcaaed7ce4de684fb793ea0cb9f6fc9800 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 13 Dec 2021 09:57:40 -0500 Subject: [PATCH 5/8] Update execution.go --- provider/schema/execution.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/provider/schema/execution.go b/provider/schema/execution.go index c2b81ae0..dd4be1c7 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -44,7 +44,7 @@ type ExecutionData struct { extraFields map[string]interface{} // partialFetch if true allows partial fetching of resources partialFetch bool - // PartialFetchFailureResult is a map of resources where the fetch process failed + // PartialFetchFailureResult is a an array of resources where the fetch process failed PartialFetchFailureResult []ResourceFetchError // partialFetchChan is the channel that is used to send failed resource fetches partialFetchChan chan ResourceFetchError @@ -220,9 +220,6 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, // check if channel iteration stopped because of resolver failure if resolverErr != nil { client.Logger().Error("received resolve resources error", "table", e.Table.Name, "error", resolverErr) - if strings.Contains(resolverErr.Error(), ": socket: too many open files") { - return 0, diag.FromError(resolverErr, diag.WARNING, diag.THROTTLE, e.ResourceName, resolverErr.Error(), "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") - } return 0, e.checkPartialFetchError(resolverErr, nil, "table resolve error") } // Print only parent resources @@ -428,6 +425,9 @@ func (e *ExecutionData) checkPartialFetchError(err error, res *Resource, customM partialFetchFailure.RootPrimaryKeyValues = root.Keys() } } + if strings.Contains(err.Error(), ": socket: too many open files") { + partialFetchFailure.Err = diag.FromError(err, diag.WARNING, diag.THROTTLE, e.ResourceName, err.Error(), "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") + } // Send information via our channel e.partialFetchChan <- partialFetchFailure return nil From 2de84709147126fb6b1eef6223b1e0062663e76e Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 13 Dec 2021 10:08:11 -0500 Subject: [PATCH 6/8] Update execution.go --- provider/schema/execution.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/provider/schema/execution.go b/provider/schema/execution.go index dd4be1c7..b45245ef 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -197,11 +197,16 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, close(res) }() err := e.Table.Resolver(ctx, client, parent, res) - if err != nil && e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { - client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) - // add partial fetch error, this will be passed in diagnostics, although it was ignored - _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") - return + if err != nil { + if strings.Contains(err.Error(), ": socket: too many open files") { + client.Logger().Warn("OS error during %s fetch. try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters", e.Table.Name) + } + if e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { + client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) + // add partial fetch error, this will be passed in diagnostics, although it was ignored + _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") + return + } } resolverErr = err }() From d6aa8469ea82a7d54f16a5b8f598248c945be6b4 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 13 Dec 2021 10:33:57 -0500 Subject: [PATCH 7/8] Update execution.go --- provider/schema/execution.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/provider/schema/execution.go b/provider/schema/execution.go index b45245ef..676395c7 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -71,6 +71,8 @@ func (p ResourceFetchError) Error() string { // partialFetchFailureBufferLength defines the buffer length for the partialFetchChan. const partialFetchFailureBufferLength = 10 +const fdLimitMessage = "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters" + // NewExecutionData Create a new execution data func NewExecutionData(db Database, logger hclog.Logger, table *Table, disableDelete bool, extraFields map[string]interface{}, partialFetch bool) ExecutionData { return ExecutionData{ @@ -199,7 +201,7 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, err := e.Table.Resolver(ctx, client, parent, res) if err != nil { if strings.Contains(err.Error(), ": socket: too many open files") { - client.Logger().Warn("OS error during %s fetch. try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters", e.Table.Name) + client.Logger().Warn("OS error during %s fetch. %s ", e.Table.Name, fdLimitMessage) } if e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) @@ -431,7 +433,8 @@ func (e *ExecutionData) checkPartialFetchError(err error, res *Resource, customM } } if strings.Contains(err.Error(), ": socket: too many open files") { - partialFetchFailure.Err = diag.FromError(err, diag.WARNING, diag.THROTTLE, e.ResourceName, err.Error(), "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters") + // Return a Diagnostic error so that it can be properly propegated back to the user via the CLI + partialFetchFailure.Err = diag.FromError(err, diag.WARNING, diag.THROTTLE, e.ResourceName, err.Error(), fdLimitMessage) } // Send information via our channel e.partialFetchChan <- partialFetchFailure From 0c806f1fe5bf7a250dc27d38a74c862c11645a00 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 13 Dec 2021 12:22:20 -0500 Subject: [PATCH 8/8] Update execution.go --- provider/schema/execution.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/provider/schema/execution.go b/provider/schema/execution.go index 676395c7..c266690e 100644 --- a/provider/schema/execution.go +++ b/provider/schema/execution.go @@ -68,10 +68,13 @@ func (p ResourceFetchError) Error() string { return p.Err.Error() } -// partialFetchFailureBufferLength defines the buffer length for the partialFetchChan. -const partialFetchFailureBufferLength = 10 +const ( + // partialFetchFailureBufferLength defines the buffer length for the partialFetchChan. + partialFetchFailureBufferLength = 10 -const fdLimitMessage = "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters" + // fdLimitMessage defines the message for when a client isn't able to fetch because the open fd limit is hit + fdLimitMessage = "try increasing number of available file descriptors via `ulimit -n 10240` or by increasing timeout via provider specific parameters" +) // NewExecutionData Create a new execution data func NewExecutionData(db Database, logger hclog.Logger, table *Table, disableDelete bool, extraFields map[string]interface{}, partialFetch bool) ExecutionData { @@ -199,16 +202,11 @@ func (e ExecutionData) callTableResolve(ctx context.Context, client ClientMeta, close(res) }() err := e.Table.Resolver(ctx, client, parent, res) - if err != nil { - if strings.Contains(err.Error(), ": socket: too many open files") { - client.Logger().Warn("OS error during %s fetch. %s ", e.Table.Name, fdLimitMessage) - } - if e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { - client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) - // add partial fetch error, this will be passed in diagnostics, although it was ignored - _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") - return - } + if err != nil && e.Table.IgnoreError != nil && e.Table.IgnoreError(err) { + client.Logger().Warn("ignored an error", "err", err, "table", e.Table.Name) + // add partial fetch error, this will be passed in diagnostics, although it was ignored + _ = e.checkPartialFetchError(err, parent, "table resolver ignored error") + return } resolverErr = err }()