Merge remote-tracking branch 'upstream/main'

* upstream/main: (22 commits) fix: Skip reattached provider update checks (cloudquery#801) fix: Add Git Binary To Docker chore(main): Release v0.24.1 (cloudquery#780) fix: Test flakiness (cloudquery#790) feat: Store Policy Output (cloudquery#709) fix: Panic in loadPolicyFromSource (cloudquery#787) fix: Panic in OsFs.downloadFile (cloudquery#789) fix: Error classifier improvements (cloudquery#788) fix: Console log log level (cloudquery#786) refactor: Use pointer for connection receiver (cloudquery#782) fix(deps): Update module github.com/cloudquery/cq-provider-sdk to v0.10.2 (cloudquery#784) refactor: Use copy instead of loop (cloudquery#781) fix: Remove always nil return value (cloudquery#778) test(fetch): Fix assertions to match new test provider (cloudquery#777) test(fetch): Pin test provider version (cloudquery#776) chore(main): Release v0.24.0 (cloudquery#749) fix: Upgrade protocol version to V5 (cloudquery#774) fix: Check policy version on it's core version (cloudquery#773) fix: Policy output file name (cloudquery#770) fix: Policy executor (cloudquery#769) ...
TinLe · May 27, 2022 · 5eceaf4 · 5eceaf4
2 parents 8e994dd + 714b446
commit 5eceaf4
Show file tree

Hide file tree

Showing 60 changed files with 1,457 additions and 1,387 deletions.
diff --git a/.github/workflows/history_test.yml b/.github/workflows/history_test.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,54 @@ All notable changes to CloudQuery will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+### [0.24.1](https://github.com/cloudquery/cloudquery/compare/v0.24.0...v0.24.1) (2022-05-26)
+
+
+### Features
+
+* Store Policy Output ([#709](https://github.com/cloudquery/cloudquery/issues/709)) ([bda4a50](https://github.com/cloudquery/cloudquery/commit/bda4a50d7ddf446f92ba72850e8d1e778620be6a))
+
+
+### Bug Fixes
+
+* Console log log level ([#786](https://github.com/cloudquery/cloudquery/issues/786)) ([8a7b76e](https://github.com/cloudquery/cloudquery/commit/8a7b76ecdeaceb62dd7b5cd376aaf8760ea5f9e3))
+* **deps:** Update module github.com/cloudquery/cq-provider-sdk to v0.10.2 ([#784](https://github.com/cloudquery/cloudquery/issues/784)) ([f904794](https://github.com/cloudquery/cloudquery/commit/f9047942373cd0650412b7eb6e9a4a1694f31596))
+* Error classifier improvements ([#788](https://github.com/cloudquery/cloudquery/issues/788)) ([f32a701](https://github.com/cloudquery/cloudquery/commit/f32a701a8893cdb2f0b5f28c304a98d341ef4f82))
+* Panic in loadPolicyFromSource ([#787](https://github.com/cloudquery/cloudquery/issues/787)) ([5953d09](https://github.com/cloudquery/cloudquery/commit/5953d091110c30f6d8760aadc1784fe9a88eff95))
+* Panic in OsFs.downloadFile ([#789](https://github.com/cloudquery/cloudquery/issues/789)) ([f2c76e8](https://github.com/cloudquery/cloudquery/commit/f2c76e8d9e95d77b479f0d12bac4bfdfebb9e15a))
+* Remove always nil return value ([#778](https://github.com/cloudquery/cloudquery/issues/778)) ([dca8745](https://github.com/cloudquery/cloudquery/commit/dca8745c24fbff70c7b3ba89eafb87b90259daeb))
+* Test flakiness ([#790](https://github.com/cloudquery/cloudquery/issues/790)) ([1497c51](https://github.com/cloudquery/cloudquery/commit/1497c511da1f31e7b0cf8eca2a9b159580bbe8e0))
+
+## [0.24.0](https://github.com/cloudquery/cloudquery/compare/v0.23.4...v0.24.0) (2022-05-24)
+
+
+### ⚠ BREAKING CHANGES
+
+* Remove provider migrations (#731)
+
+### Features
+
+* Classify some policy download errors as USER ([#742](https://github.com/cloudquery/cloudquery/issues/742)) ([8224e60](https://github.com/cloudquery/cloudquery/commit/8224e60d3a76d3b3f181d3b32b9153a63b04816a))
+* Remove provider migrations ([#731](https://github.com/cloudquery/cloudquery/issues/731)) ([bb93967](https://github.com/cloudquery/cloudquery/commit/bb93967803a28dfc1fccc0fb45af207d76208b2d))
+
+
+### Bug Fixes
+
+* Added root policy config inheritance to selected subpolicy ([#702](https://github.com/cloudquery/cloudquery/issues/702)) ([0d8b3c7](https://github.com/cloudquery/cloudquery/commit/0d8b3c77741177e6417c59e269856639dce0155e))
+* Check policy version on it's core version ([#773](https://github.com/cloudquery/cloudquery/issues/773)) ([c7c9ad0](https://github.com/cloudquery/cloudquery/commit/c7c9ad0384e2b90c6507c52ab87da65db2bc836f))
+* Classify "no policies in config" as USER error ([#743](https://github.com/cloudquery/cloudquery/issues/743)) ([4cbc03e](https://github.com/cloudquery/cloudquery/commit/4cbc03e22f5a0bbfa33812b407e65704727a88fd))
+* **deps:** Update SDK ([#758](https://github.com/cloudquery/cloudquery/issues/758)) ([c9b4094](https://github.com/cloudquery/cloudquery/commit/c9b4094cfe2ebbae9b6cbbc8df4568c7460208e5))
+* Diag import clean up ([#744](https://github.com/cloudquery/cloudquery/issues/744)) ([437c956](https://github.com/cloudquery/cloudquery/commit/437c956ee941c3ed2c9859a0af6a0b88401b58d8))
+* Don't attempt to download provider in re-attach mode ([#748](https://github.com/cloudquery/cloudquery/issues/748)) ([59973b8](https://github.com/cloudquery/cloudquery/commit/59973b84826599915f7b76fc8d8b16626dd26c74))
+* FetchId column regression ([#745](https://github.com/cloudquery/cloudquery/issues/745)) ([585d395](https://github.com/cloudquery/cloudquery/commit/585d39589ef6c27ae2aab5d224fc00a2387d7628))
+* Handle DeadlineExceeded errors ([#741](https://github.com/cloudquery/cloudquery/issues/741)) ([0167ce4](https://github.com/cloudquery/cloudquery/commit/0167ce4158d4795fc3a4b0f6661c19ae197c20c9))
+* Handle Outputting Policies With Selectors ([a3ecfc9](https://github.com/cloudquery/cloudquery/commit/a3ecfc9166170e1bb77011befd11a5fbe1c86007))
+* Policy executor ([#769](https://github.com/cloudquery/cloudquery/issues/769)) ([d5b6aef](https://github.com/cloudquery/cloudquery/commit/d5b6aef25f1cccaaf30618c53d5d7204f83d74aa))
+* Policy output file name ([#770](https://github.com/cloudquery/cloudquery/issues/770)) ([1a87c25](https://github.com/cloudquery/cloudquery/commit/1a87c259ddf9d0d1694976f551503918ca1557bd))
+* Space trimming in telemetry file ([#734](https://github.com/cloudquery/cloudquery/issues/734)) ([16c4cfc](https://github.com/cloudquery/cloudquery/commit/16c4cfce7e15f4474af3ab5d7e0cdb3698d2d08e))
+* Upgrade protocol version to V5 ([#774](https://github.com/cloudquery/cloudquery/issues/774)) ([69b405d](https://github.com/cloudquery/cloudquery/commit/69b405d686031dec3443ebc018047f42dd259d0e))
+* Use consistent descriptions for flags ([#753](https://github.com/cloudquery/cloudquery/issues/753)) ([cedeb3d](https://github.com/cloudquery/cloudquery/commit/cedeb3d0b1d733b352a03d45753914a6e1d11ee4))
+
 ### [0.23.4](https://github.com/cloudquery/cloudquery/compare/v0.23.3...v0.23.4) (2022-05-17)
 
 

diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser
@@ -1,4 +1,5 @@
 FROM alpine:latest
+RUN apk --no-cache add git
 ENTRYPOINT ["/app/cloudquery"]
 WORKDIR /app
 COPY cloudquery .
diff --git a/cmd/fetch.go b/cmd/fetch.go
@@ -69,9 +69,7 @@ func filterConfigProviders(list []string) func(*config.Config) error {
 			if len(parts) == 2 && parts[1] != "*" {
 				resources := strings.Split(parts[1], ",")
 				pMap[prov] = make([]string, len(resources))
-				for i, res := range resources {
-					pMap[prov][i] = res
-				}
+				copy(pMap[prov], resources)
 			} else {
 				pMap[prov] = nil
 			}

diff --git a/cmd/policy.go b/cmd/policy.go
@@ -8,8 +8,9 @@ import (
 
 	"github.com/spf13/viper"
 
-	"github.com/cloudquery/cloudquery/pkg/ui/console"
 	"github.com/spf13/cobra"
+
+	"github.com/cloudquery/cloudquery/pkg/ui/console"
 )
 
 const (
@@ -20,6 +21,7 @@ const (
 	policySnapshotHelpMsg = `Take database snapshot of all tables included in a CloudQuery policy`
 	policyTestHelpMsg     = "Tests policy against a precompiled set of database snapshots"
 	policyValidateHelpMsg = "Validate policy for any issues and diagnostics"
+	policyPruneHelpMsg    = "Prune policy executions from the database which are older than the relative time specified"
 )
 
 var (
@@ -75,6 +77,7 @@ var (
 
 	outputDir    string
 	noResults    bool
+	storeResults bool
 	policyRunCmd = &cobra.Command{
 		Use:   "run",
 		Short: policyRunHelpMsg,
@@ -100,7 +103,7 @@ var (
 			if len(args) == 1 {
 				source = args[0]
 			}
-			diags := c.RunPolicies(ctx, source, outputDir, noResults)
+			diags := c.RunPolicies(ctx, source, outputDir, noResults, storeResults)
 			errors.CaptureDiagnostics(diags, map[string]string{"command": "policy_run"})
 			if diags.HasErrors() {
 				return fmt.Errorf("policy has one or more errors, check logs")
@@ -149,6 +152,25 @@ var (
 			return fmt.Errorf("policy validate has one or more errors, check logs")
 		}),
 	}
+
+	prunePolicyCmd = &cobra.Command{
+		Use:   "prune",
+		Short: policyPruneHelpMsg,
+		Long:  policyPruneHelpMsg,
+		Example: `
+  # Prune the policy executions which are older than the relative time specified
+  cloudquery policy prune 24h`,
+		Args: cobra.ExactArgs(1),
+		Run: handleCommand(func(ctx context.Context, c *console.Client, cmd *cobra.Command, args []string) error {
+			retentionPeriod := args[0]
+			diags := c.PrunePolicyExecutions(ctx, retentionPeriod)
+			errors.CaptureDiagnostics(diags, map[string]string{"command": "policy_prune"})
+			if diags.HasErrors() {
+				return fmt.Errorf("policy prune has one or more errors, check logs")
+			}
+			return nil
+		}),
+	}
 )
 
 func init() {
@@ -160,6 +182,7 @@ func init() {
 	flags := policyRunCmd.Flags()
 	flags.StringVar(&outputDir, "output-dir", "", "Generates a new file for each policy at the given dir with the output")
 	flags.BoolVar(&noResults, "no-results", false, "Do not show policies results")
+	flags.BoolVar(&storeResults, "enable-db-persistence", false, "Enable storage of policy output in database")
 	flags.Bool("disable-fetch-check", false, "Disable checking if a respective fetch happened before running policies")
 	_ = viper.BindPFlag("disable-fetch-check", flags.Lookup("disable-fetch-check"))
 	policyRunCmd.SetUsageTemplate(usageTemplateWithFlags)
@@ -176,6 +199,8 @@ func init() {
 
 	policyCmd.AddCommand(validatePolicyCmd)
 
+	policyCmd.AddCommand(prunePolicyCmd)
+
 	policyCmd.SetUsageTemplate(usageTemplateWithFlags)
 	rootCmd.AddCommand(policyCmd)
 }
diff --git a/cmd/provider.go b/cmd/provider.go
@@ -5,13 +5,11 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/cloudquery/cloudquery/pkg/core"
 	"github.com/cloudquery/cloudquery/pkg/errors"
-	"github.com/cloudquery/cq-provider-sdk/provider/diag"
-
 	"github.com/cloudquery/cloudquery/pkg/ui"
-
-	"github.com/cloudquery/cloudquery/pkg/core"
 	"github.com/cloudquery/cloudquery/pkg/ui/console"
+	"github.com/cloudquery/cq-provider-sdk/provider/diag"
 
 	"github.com/spf13/cobra"
 )

diff --git a/cmd/root.go b/cmd/root.go
@@ -5,14 +5,12 @@ import (
 	"os"
 	"strings"
 
-	"github.com/google/uuid"
-
 	"github.com/cloudquery/cloudquery/internal/analytics"
-
 	"github.com/cloudquery/cloudquery/internal/logging"
 	"github.com/cloudquery/cloudquery/pkg/core"
 	"github.com/cloudquery/cloudquery/pkg/ui"
 
+	"github.com/google/uuid"
 	zerolog "github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -123,6 +121,7 @@ func init() {
 	rootCmd.PersistentFlags().String("policy-dir", "", "set policies directory (env: CQ_POLICY_DIR)")
 	rootCmd.PersistentFlags().String("reattach-providers", "", "path to reattach unmanaged plugins, mostly used for testing purposes (env: CQ_REATTACH_PROVIDERS)")
 	rootCmd.PersistentFlags().Bool("skip-build-tables", false, "enable skipping building tables. Should only be set if tables already exist")
+	rootCmd.PersistentFlags().Bool("force-drop", false, "when upgrading schema, force dropping of any dependent views")
 
 	rootCmd.PersistentFlags().Bool("no-telemetry", false, "disable telemetry collection")
 	rootCmd.PersistentFlags().Bool("inspect-telemetry", false, "enable telemetry inspection")
@@ -143,6 +142,7 @@ func init() {
 	_ = viper.BindPFlag("configPath", rootCmd.PersistentFlags().Lookup("config"))
 	_ = viper.BindPFlag("no-verify", rootCmd.PersistentFlags().Lookup("no-verify"))
 	_ = viper.BindPFlag("skip-build-tables", rootCmd.PersistentFlags().Lookup("skip-build-tables"))
+	_ = viper.BindPFlag("force-drop", rootCmd.PersistentFlags().Lookup("force-drop"))
 
 	// Telemetry specific options
 	_ = viper.BindPFlag("no-telemetry", rootCmd.PersistentFlags().Lookup("no-telemetry"))

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
-	github.com/cloudquery/cq-provider-sdk v0.9.5
+	github.com/cloudquery/cq-provider-sdk v0.10.3
 	github.com/fatih/color v1.13.0
 	github.com/golang-migrate/migrate/v4 v4.15.0
 	github.com/google/go-github/v35 v35.2.0
@@ -32,7 +32,6 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f
 	github.com/aws/aws-sdk-go v1.43.41
-	github.com/creasty/defaults v1.5.2
 	github.com/doug-martin/goqu/v9 v9.17.0
 	github.com/driftprogramming/pgxpoolmock v1.1.0
 	github.com/georgysavva/scany v0.2.9
@@ -95,6 +94,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
 	github.com/aws/smithy-go v1.11.2 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+	github.com/creasty/defaults v1.5.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/gofrs/uuid v4.0.0+incompatible // indirect

diff --git a/go.sum b/go.sum
@@ -350,8 +350,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
-github.com/cloudquery/cq-provider-sdk v0.9.5 h1:mDPowXQNIvYY/yBpXTGSVTPa05tv/YpaeI1kglzjqpI=
-github.com/cloudquery/cq-provider-sdk v0.9.5/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ=
+github.com/cloudquery/cq-provider-sdk v0.10.3 h1:crVsLfXkc+XeGKq6DthI2WeyRtibNrypZyK4MwP+Zv4=
+github.com/cloudquery/cq-provider-sdk v0.10.3/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ=
 github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec=
 github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=

diff --git a/internal/file/file.go b/internal/file/file.go
@@ -51,10 +51,10 @@ func (o *OsFs) downloadFile(ctx context.Context, filepath, url string, progressU
 	// Create the file, but give it a tmp file extension, this means we won't overwrite a
 	// file until it's downloaded, but we'll remove the tmp extension once downloaded.
 	out, err := o.fs.Create(filepath + ".tmp")
-	defer func() { _ = out.Close() }()
 	if err != nil {
 		return err
 	}
+	defer func() { _ = out.Close() }()
 	// Get the data
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {

diff --git a/internal/file/file_test.go b/internal/file/file_test.go
@@ -22,17 +22,30 @@ func TestOsFs_DownloadFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	ctx := context.Background()
-	filePath := filepath.Join(testDir, "cloudquery_linux.zip")
-	if err := osFs.DownloadFile(ctx, filePath, testFileDownloadUrl, nil); err != nil {
-		t.Fatal(err)
-	}
 
-	// With progress updater
-	pu := console.NewProgress(ctx, func(o *console.ProgressOptions) {
-		o.AppendDecorators = []decor.Decorator{decor.Percentage()}
+	t.Run("successful run without progress bar", func(t *testing.T) {
+		filePath := filepath.Join(testDir, "cloudquery_linux.zip")
+		if err := osFs.DownloadFile(ctx, filePath, testFileDownloadUrl, nil); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("successful run with progress bar", func(t *testing.T) {
+		pu := console.NewProgress(ctx, func(o *console.ProgressOptions) {
+			o.AppendDecorators = []decor.Decorator{decor.Percentage()}
+		})
+		progressCB := ui.CreateProgressUpdater(pu, "test-123")
+		filePath := filepath.Join(testDir, "cloudquery_linux.zip")
+		if err := osFs.DownloadFile(ctx, filePath, testFileDownloadUrl, progressCB); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("failure with non existing path", func(t *testing.T) {
+		// we have to use real fs because mem fs creates non existing folders on the fly
+		osFs.SetFSInstance(afero.NewOsFs())
+		if err := osFs.DownloadFile(ctx, "/no/such/dir/cloudquery_linux.zip", testFileDownloadUrl, nil); err == nil {
+			t.Fatal("expected to get non nil error")
+		}
 	})
-	progressCB := ui.CreateProgressUpdater(pu, "test-123")
-	if err := osFs.DownloadFile(ctx, filePath, testFileDownloadUrl, progressCB); err != nil {
-		t.Fatal(err)
-	}
 }
diff --git a/internal/maps.go b/internal/maps.go
@@ -0,0 +1,21 @@
+package internal
+
+// FlattenRow flattens a row map and skips empty columns
+func FlattenRow(m map[string]interface{}) map[string]interface{} {
+	o := make(map[string]interface{})
+	for k, v := range m {
+		switch child := v.(type) {
+		case map[string]interface{}:
+			nm := FlattenRow(child)
+			for nk, nv := range nm {
+				o[nk] = nv
+			}
+		default:
+			if v == nil {
+				continue
+			}
+			o[k] = v
+		}
+	}
+	return o
+}