test: Refactor integration tests (#450)

* test: Refactor integration tests

All integration tests now run from TestIntegration.
This makes the tests faster and remove huge amount of connection
to the database.

Also remove bash magic to test for warnings using grep and do
all the checks on the SDK level.

* chore: Remove old lint.yml

Use only golang_lint.yml and doc_validation.yml that are
managed centerally under cloudquery/.github repo

* chore: update to cq-provider-sdk v0.7.1

* chore: fix golanglint

* test: Add CQ_TEST_CFG to override config in tests

* test: Remove !integration tag

* chore: Remove cq_fetch.yml

* chore: Rename int_test.yml to test_integration.yml

.github/workflows/int_test.yml → .github/workflows/test_integration.yml

@@ -1,4 +1,5 @@
-name: integration_test
+# DONT EDIT. This file is synced from https://github.com/cloudquery/.github/.github
+name: test_integration
 
 on:
   push:
@@ -10,7 +11,6 @@ on:
 jobs:
   is_organization_member:
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target'
     outputs:
       status: ${{ steps.is_organization_member.result }}
     steps:
@@ -22,10 +22,10 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}  
       - name: Skip if not member and event is pull_request_target
         run: |
-            echo "user is not part of org. Please run 'go test -tags=integration ./..'"
+            echo "user is not part of org. Please run 'go test -run=TestIntegration -tags=integration ./..'"
             echo "and paste the output in the PR"
 
-  integration_test:
+  test_integration:
     needs: is_organization_member
     if: github.event_name != 'pull_request_target' || needs.is_organization_member.outputs.status == 'true'
     strategy:
@@ -76,7 +76,7 @@ jobs:
 
       - name: Integration Test
         run: |
-          go test -tags=integration ./...
+          go test -run=TestIntegration -tags=integration ./...
 
       - name: Slack Notification
         uses: rtCamp/action-slack-notify@v2

client/logger.go

@@ -1,6 +1,8 @@
 package client
 
-import "github.com/hashicorp/go-hclog"
+import (
+	"github.com/hashicorp/go-hclog"
+)
 
 type awsLogger struct {
 	hclog.Logger

client/testing.go

@@ -51,9 +51,7 @@ func AwsMockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.
 				return &Config{}
 			},
 		},
-		Table:          table,
-		Config:         cfg,
-		SkipEmptyJsonB: options.SkipEmptyJsonB,
+		Config: cfg,
 	})
 }
 
@@ -75,7 +73,6 @@ func AWSTestHelper(t *testing.T, table *schema.Table) {
 				"test_resource": table,
 			},
 		},
-		Table:  table,
 		Config: cfg,
 	})
 

go.mod

@@ -59,7 +59,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/wafv2 v1.5.1
 	github.com/aws/smithy-go v1.9.1
 	github.com/bxcodec/faker v2.0.1+incompatible
-	github.com/cloudquery/cq-provider-sdk v0.7.0
+	github.com/cloudquery/cq-provider-sdk v0.7.1
 	github.com/cloudquery/faker/v3 v3.7.5
 	github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8
 	github.com/golang/mock v1.6.0
@@ -145,4 +145,4 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-)
+)

go.sum

@@ -320,6 +320,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cloudquery/cq-provider-sdk v0.7.0 h1:PWERMQi7oUW0rHn/qKvf7cVcOhMB9xgx/i0W3hvoXOc=
 github.com/cloudquery/cq-provider-sdk v0.7.0/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs=
+github.com/cloudquery/cq-provider-sdk v0.7.1 h1:thKsjnBbeSAXO4nodYR9OFrdXVB243SXptlwp0RQ7ok=
+github.com/cloudquery/cq-provider-sdk v0.7.1/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs=
 github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
 github.com/cloudquery/faker/v3 v3.7.5 h1:G7ANdEEcm8TvAAjIwNWSLrYK36CFCiSlrCqOTGCccL0=
 github.com/cloudquery/faker/v3 v3.7.5/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=