From 5ccec20798d244a888a8b598481aff530c4d650b Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Wed, 29 Dec 2021 12:09:42 +0200 Subject: [PATCH] snapshot testing sdk (#137) Change `TestResource` both for mock testing and integration tests --- CHANGELOG.md | 4 + go.mod | 3 +- provider/provider.go | 6 +- provider/provider_test.go | 2 +- provider/schema/column.go | 3 + provider/schema/resource.go | 10 + provider/testing/integration.go | 483 -------------------------------- provider/testing/integration.md | 38 --- provider/testing/resource.go | 213 +++++++------- testlog/testlog.go | 137 +++++++++ 10 files changed, 277 insertions(+), 622 deletions(-) delete mode 100644 provider/testing/integration.go delete mode 100644 provider/testing/integration.md create mode 100644 testlog/testlog.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a682ba6..91090ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v0.6.0-beta] - 2021-12-20 +### :gear: Changed +* **Breaking Change**: changed `TestResource` API [#137](https://github.com/cloudquery/cq-provider-sdk/pull/137) + ## [v0.5.7]- 2021-12-20 ### :gear: Changed diff --git a/go.mod b/go.mod index 8b6a7d3e..cc27b134 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/gofrs/uuid v4.0.0+incompatible github.com/golang-migrate/migrate/v4 v4.15.0 github.com/golang/mock v1.6.0 + github.com/google/go-cmp v0.5.6 github.com/google/uuid v1.3.0 github.com/hashicorp/go-hclog v1.0.0 github.com/hashicorp/go-plugin v1.4.3 @@ -25,6 +26,7 @@ require ( github.com/jackc/pgx/v4 v4.13.0 github.com/mitchellh/hashstructure v1.1.0 github.com/modern-go/reflect2 v1.0.2 + github.com/sergi/go-diff v1.2.0 github.com/spf13/afero v1.6.0 github.com/spf13/cast v1.4.1 github.com/stretchr/testify v1.7.0 @@ -46,7 +48,6 @@ require ( github.com/fatih/color v1.13.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.6 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/terraform-json v0.13.0 // indirect diff --git a/provider/provider.go b/provider/provider.go index e87b2108..3ebd6e15 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -94,10 +94,10 @@ func (p *Provider) GetProviderConfig(_ context.Context, _ *cqproto.GetProviderCo func (p *Provider) ConfigureProvider(_ context.Context, request *cqproto.ConfigureProviderRequest) (*cqproto.ConfigureProviderResponse, error) { if p.meta != nil { - return &cqproto.ConfigureProviderResponse{Error: fmt.Sprintf("provider %s was already configured", p.Name)}, nil + return &cqproto.ConfigureProviderResponse{Error: fmt.Sprintf("provider %s was already configured", p.Name)}, fmt.Errorf("provider %s was already configured", p.Name) } if p.Logger == nil { - return &cqproto.ConfigureProviderResponse{Error: fmt.Sprintf("provider %s logger not defined, make sure to run it with serve", p.Name)}, nil + return &cqproto.ConfigureProviderResponse{Error: fmt.Sprintf("provider %s logger not defined, make sure to run it with serve", p.Name)}, fmt.Errorf("provider %s logger not defined, make sure to run it with serve", p.Name) } // set database creator if p.databaseCreator == nil { @@ -116,7 +116,7 @@ func (p *Provider) ConfigureProvider(_ context.Context, request *cqproto.Configu // if we received an empty config we notify in log and only use defaults. if len(request.Config) == 0 { p.Logger.Info("Received empty configuration, using only defaults") - } else if err := hclsimple.Decode("config.json", request.Config, nil, providerConfig); err != nil { + } else if err := hclsimple.Decode("config.hcl", request.Config, nil, providerConfig); err != nil { p.Logger.Error("Failed to load configuration.", "error", err) return &cqproto.ConfigureProviderResponse{}, err } diff --git a/provider/provider_test.go b/provider/provider_test.go index 841851d1..e89b92ea 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -263,7 +263,7 @@ func TestProvider_ConfigureProvider(t *testing.T) { ExtraFields: nil, }) assert.Equal(t, "provider unitest logger not defined, make sure to run it with serve", resp.Error) - assert.Nil(t, err) + assert.NotNil(t, err) // set logger this time tp.Logger = hclog.Default() resp, err = tp.ConfigureProvider(context.Background(), &cqproto.ConfigureProviderRequest{ diff --git a/provider/schema/column.go b/provider/schema/column.go index 7dda7816..0b4e62d6 100644 --- a/provider/schema/column.go +++ b/provider/schema/column.go @@ -161,6 +161,9 @@ type Column struct { IgnoreError IgnoreErrorFunc // Creation options allow modifying how column is defined when table is created CreationOptions ColumnCreationOptions + // IgnoreInIntTests if true tests integration tests wont compare this column + // in snapshot testing. Usually used for columns like: meta, last_updated + IgnoreInIntTests bool // meta holds serializable information about the column's resolvers and functions meta *ColumnMeta diff --git a/provider/schema/resource.go b/provider/schema/resource.go index 3f9a3bbf..93160e4c 100644 --- a/provider/schema/resource.go +++ b/provider/schema/resource.go @@ -3,6 +3,7 @@ package schema import ( "crypto" "fmt" + "strings" "github.com/mitchellh/hashstructure" "github.com/thoas/go-funk" @@ -105,6 +106,15 @@ func (r *Resource) GenerateCQId() error { return nil } +func (r Resource) getColumnByName(column string) *Column { + for _, c := range r.table.Columns { + if strings.Compare(column, c.Name) == 0 { + return &c + } + } + return nil +} + func hashUUID(objs interface{}) (uuid.UUID, error) { // Use SHA1 because it's fast and is reasonably enough protected against accidental collisions. // There is no scenario here where intentional created collisions could do harm. diff --git a/provider/testing/integration.go b/provider/testing/integration.go deleted file mode 100644 index d3e52805..00000000 --- a/provider/testing/integration.go +++ /dev/null @@ -1,483 +0,0 @@ -package testing - -import ( - "context" - "encoding/json" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/Masterminds/squirrel" - "github.com/cloudquery/cq-provider-sdk/cqproto" - "github.com/cloudquery/cq-provider-sdk/logging" - "github.com/cloudquery/cq-provider-sdk/provider" - "github.com/cloudquery/cq-provider-sdk/provider/schema" - "github.com/georgysavva/scany/pgxscan" - "github.com/go-test/deep" - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2/hclwrite" - "github.com/hashicorp/terraform-exec/tfexec" - "github.com/tmccombs/hcl2json/convert" -) - -const ( - tfDir = "./.test/" - infraFilesDir = "./infra/" -) - -var ( - // Make a Regex to say we only want letters and numbers - tfVarRegex = regexp.MustCompile("[^a-zA-Z0-9]+") -) - -type ResourceIntegrationTestData struct { - Table *schema.Table - Config interface{} - Resources []string - Configure func(logger hclog.Logger, data interface{}) (schema.ClientMeta, error) - Suffix string - Prefix string - VerificationBuilder func(res *ResourceIntegrationTestData) ResourceIntegrationVerification - workdir string -} - -// ResourceIntegrationVerification - a set of verification rules to query and check test related data -type ResourceIntegrationVerification struct { - Name string - ForeignKeyName string - ExpectedValues []ExpectedValue - Filter func(sq squirrel.SelectBuilder, res *ResourceIntegrationTestData) squirrel.SelectBuilder - Relations []*ResourceIntegrationVerification -} - -// ExpectedValue - describes the data that expected to be in database after fetch -type ExpectedValue struct { - Count int // expected count of items - Data map[string]interface{} // expected data of items -} - -// IntegrationTest - creates resources using terraform, fetches them to db and compares with expected values -func IntegrationTest(t *testing.T, providerCreator func() *provider.Provider, resource ResourceIntegrationTestData) { - t.Parallel() - - // prepare terraform variables - hostname, err := os.Hostname() - if err != nil { - t.Fatal(err) - } - - // whether want TF to apply and create resources instead of running a fetch on existing resources - var tfApplyResources = getEnv("TF_APPLY_RESOURCES", "") == "1" - var varPrefix = simplifyString(resource.Table.Name) - var varSuffix = simplifyString(hostname) - - prefix := getEnv("TF_VAR_PREFIX", "") - if prefix != "" { - varPrefix = prefix - } else if !tfApplyResources { - t.Fatalf("Missing resource TF_VAR_PREFIX either set this environment variable or use TF_APPLY_RESOURCES=1") - } - - suffix := getEnv("TF_VAR_SUFFIX", "") - if suffix != "" { - varSuffix = suffix - } else if !tfApplyResources { - t.Fatalf("Missing resource TF_VAR_SUFFIX either set this environment or use TF_APPLY_RESOURCES=1") - } - - // finally set picked prefix/suffix to resource - resource.Prefix = varPrefix - resource.Suffix = varSuffix - - if tfApplyResources { - tf, err := setup(&resource) - if err != nil { - t.Fatal(err) - } - - teardown, err := deploy(tf, &resource) - if teardown != nil && getEnv("TF_NO_DESTROY", "") != "1" { - defer func() { - if err = teardown(); err != nil { - t.Fatal(err) - } - }() - } else { - defer func() { - log.Printf("%s RESOURCES WERE NOT DESTROYTED. destroy them manually.\n", resource.Table.Name) - }() - } - if err != nil { - t.Fatal(err) - } - } - - log.Printf("%s verify fields\n", resource.Table.Name) - pool, err := setupDatabase() - if err != nil { - t.Fatal(err) - } - ctx := context.Background() - conn, err := pool.Acquire(ctx) - if err != nil { - t.Fatal(err) - } - defer conn.Release() - - l := logging.New(hclog.DefaultOptions) - tableCreator := provider.NewTableCreator(l) - if err := tableCreator.CreateTable(context.Background(), conn, resource.Table, nil); err != nil { - assert.FailNow(t, fmt.Sprintf("failed to create tables %s", resource.Table.Name), err) - } - - if err = fetch(providerCreator, &resource); err != nil { - t.Fatal(err) - } - - if err = verifyFields(resource, conn); err != nil { - t.Fatal(err) - } - - if err := conn.Conn().Close(ctx); err != nil { - t.Fatal(err) - } -} - -// setup - puts *.tf files into isolated test dir and creates the instance of terraform -func setup(resource *ResourceIntegrationTestData) (*tfexec.Terraform, error) { - var err error - if resource.Resources != nil { - resource.workdir, err = copyTfFiles(resource.Table.Name, resource.Resources...) - } else { - resource.workdir, err = copyTfFiles(resource.Table.Name, fmt.Sprintf("%s.tf", resource.Table.Name)) - } - - if err != nil { - // remove workdir - if e := os.RemoveAll(resource.workdir); e != nil { - return nil, fmt.Errorf("failed to RemoveAll after: %w\n reason:%s", err, e) - } - return nil, err - } - - lookPath := getEnv("TF_EXEC_PATH", "") - if lookPath == "" { - lookPath = "terraform" - } - execPath, err := exec.LookPath(lookPath) - if err != nil { - return nil, err - } - tf, err := tfexec.NewTerraform(resource.workdir, execPath) - if err != nil { - return nil, err - } - if err = enableTerraformLog(tf, resource.workdir); err != nil { - return nil, err - } - return tf, nil -} - -// deploy - uses terraform to deploy resources and builds teardown function. deployment timeout can be set via TF_EXEC_TIMEOUT env variable -func deploy(tf *tfexec.Terraform, resource *ResourceIntegrationTestData) (func() error, error) { - testSuffix := fmt.Sprintf("test_suffix=%s", resource.Suffix) - testPrefix := fmt.Sprintf("test_prefix=%s", resource.Prefix) - - teardown := func() error { - log.Printf("%s destroy\n", resource.Table.Name) - err := tf.Destroy(context.Background(), tfexec.Var(testPrefix), - tfexec.Var(testSuffix)) - if err != nil { - return err - } - log.Printf("%s cleanup\n", resource.Table.Name) - if err := os.RemoveAll(resource.workdir); err != nil { - return err - } - log.Printf("%s done\n", resource.Table.Name) - return nil - } - - ctx := context.Background() - if i, err := strconv.Atoi(getEnv("TF_EXEC_TIMEOUT", "")); err == nil { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Duration(i)*time.Minute) - defer cancel() - } - - log.Printf("%s tf init\n", resource.Table.Name) - if err := tf.Init(ctx, tfexec.Upgrade(true)); err != nil { - return teardown, err - } - - ticker := time.NewTicker(5 * time.Minute) - startTime := time.Now() - applyDone := make(chan bool) - go func() { - for { - select { - case <-ctx.Done(): - case <-applyDone: - return - case timestamp := <-ticker.C: - log.Printf("%s applying for %.2f minutes", resource.Table.Name, timestamp.Sub(startTime).Minutes()) - } - } - }() - - log.Printf("%s tf apply -var test_prefix=%s -var test_suffix=%s\n", resource.Table.Name, resource.Prefix, resource.Suffix) - err := tf.Apply(ctx, tfexec.Var(testPrefix), tfexec.Var(testSuffix)) - applyDone <- true - if err != nil { - return teardown, err - } - - return teardown, nil -} - -// fetch - fetches resources from the cloud and puts them into database. database config can be specified via DATABASE_URL env variable -func fetch(providerCreator func() *provider.Provider, resource *ResourceIntegrationTestData) error { - log.Printf("%s fetch resources\n", resource.Table.Name) - testProvider := providerCreator() - testProvider.Logger = logging.New(hclog.DefaultOptions) - - // generate a config for provider - f := hclwrite.NewFile() - f.Body().AppendBlock(gohcl.EncodeAsBlock(resource.Config, "configuration")) - data, err := convert.Bytes(f.Bytes(), "config.json", convert.Options{}) - if err != nil { - return err - } - c := map[string]interface{}{} - _ = json.Unmarshal(data, &c) - data, err = json.Marshal(c["configuration"].([]interface{})[0]) - if err != nil { - return err - } - - testProvider.Configure = resource.Configure - if _, err = testProvider.ConfigureProvider(context.Background(), &cqproto.ConfigureProviderRequest{ - CloudQueryVersion: "", - Connection: cqproto.ConnectionDetails{DSN: getEnv("DATABASE_URL", - "host=localhost user=postgres password=pass DB.name=postgres port=5432")}, - Config: data, - DisableDelete: true, - }); err != nil { - return err - } - - var resourceSender = &fakeResourceSender{ - Errors: []string{}, - } - - if err = testProvider.FetchResources(context.Background(), - &cqproto.FetchResourcesRequest{ - Resources: []string{findResourceFromTableName(resource.Table, testProvider.ResourceMap)}, - }, - resourceSender, - ); err != nil { - return err - } - - if len(resourceSender.Errors) > 0 { - return fmt.Errorf("error/s occur during test, %s", strings.Join(resourceSender.Errors, ", ")) - } - - return nil -} - -// enableTerraformLog - sets the path for terraform log files for current test -func enableTerraformLog(tf *tfexec.Terraform, workdir string) error { - abs, err := filepath.Abs(workdir) - if err != nil { - return err - } - dst := filepath.Join(abs, string(os.PathSeparator), "tflog") - if _, err = os.Create(dst); err != nil { - return err - } - if err = tf.SetLogPath(dst); err != nil { - return err - } - tf.SetLogger(log.Default()) - return nil -} - -// verifyFields - gets the root db entry and check all its expected relations -func verifyFields(resource ResourceIntegrationTestData, conn pgxscan.Querier) error { - verification := resource.VerificationBuilder(&resource) - - // build query to get the root object - sq := squirrel.StatementBuilder. - PlaceholderFormat(squirrel.Dollar). - Select(fmt.Sprintf("json_agg(%s)", verification.Name)). - From(verification.Name) - // use special filter if it is set. - if verification.Filter != nil { - sq = verification.Filter(sq, &resource) - } else { - sq = sq.Where(squirrel.Eq{"tags->>'TestId'": resource.Suffix}) - } - query, args, err := sq.ToSql() - if err != nil { - return fmt.Errorf("%s -> %w", verification.Name, err) - } - - var data []map[string]interface{} - if err := pgxscan.Get(context.Background(), conn, &data, query, args...); err != nil { - return fmt.Errorf("%s -> %w", verification.Name, err) - } - - if err = compareDataWithExpected(verification.ExpectedValues, data); err != nil { - return fmt.Errorf("verification failed for table %s; %w", resource.Table.Name, err) - } - - // verify root entry relations - for _, e := range data { - id, ok := e["cq_id"] - if !ok { - return fmt.Errorf("failed to get parent id for %s", resource.Table.Name) - } - if err = verifyRelations(verification.Relations, id, resource.Table.Name, conn); err != nil { - return fmt.Errorf("verification failed for relations of table entry %s; cq_id: %v -> %w", resource.Table.Name, id, err) - } - } - return nil -} - -// verifyRelations - recursively runs through all the relations and compares their values with expected data -func verifyRelations(relations []*ResourceIntegrationVerification, parentId interface{}, parentName string, conn pgxscan.Querier) error { - for _, relation := range relations { - // build query to get relation - sq := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar). - Select(fmt.Sprintf("json_agg(%s)", relation.Name)). - From(relation.Name). - LeftJoin(fmt.Sprintf("%[1]s on %[1]s.cq_id = %[3]s.%[2]s", parentName, relation.ForeignKeyName, relation.Name)). - Where(squirrel.Eq{fmt.Sprintf("%s.cq_id", parentName): parentId}) - query, args, err := sq.ToSql() - - if err != nil { - return fmt.Errorf("%s -> %w", relation.Name, err) - } - - var data []map[string]interface{} - if err := pgxscan.Get(context.Background(), conn, &data, query, args...); err != nil { - return fmt.Errorf("%s -> %w", relation.Name, err) - } - - if err = compareDataWithExpected(relation.ExpectedValues, data); err != nil { - return fmt.Errorf("%s -> %w", relation.Name, err) - } - - // verify relation entry relations - for _, e := range data { - id, ok := e["cq_id"] - if !ok { - return fmt.Errorf("failed to get parent id for %s", relation.Name) - } - if err = verifyRelations(relation.Relations, id, relation.Name, conn); err != nil { - return fmt.Errorf("%s cq_id: %v -> %w", relation.Name, id, err) - } - } - } - return nil -} - -// compareDataWithExpected - runs through expected values and checks if they are satisfied with received data -func compareDataWithExpected(expected []ExpectedValue, received []map[string]interface{}) error { - var errors []error - // clone []map that will be compared - toCompare := make([]map[string]interface{}, len(received)) - for i, row := range received { - toCompare[i] = make(map[string]interface{}) - for key, value := range row { - toCompare[i][key] = value - } - } - - for _, verification := range expected { - found := 0 - for i := 0; i < len(toCompare); i++ { - if toCompare[i] == nil { - continue // this row is already verified - skip - } - err := compareData(verification.Data, toCompare[i]) - if err != nil { - errors = append(errors, err) - continue - } - toCompare[i] = nil // row passed verification - it won't be used - found++ - } - // verification.Count == 0 means we expect at least 1 result - if verification.Count == 0 && found > 0 { - continue - } - - if verification.Count != found { - return fmt.Errorf("expected to have %d but got %d entries with one of the %v\nerrors: %v", verification.Count, found, verification.Data, errors) - } - } - return nil -} - -// compareData - checks if the second argument has all the entries of the first argument. arguments are jsons - map[string]interface{} -func compareData(verification, row map[string]interface{}) error { - for k, v := range verification { - diff := deep.Equal(row[k], v) - if diff != nil { - return fmt.Errorf("%+v", diff) - } - } - return nil -} - -// simplifyString - prepares the string to be used in resources names -func simplifyString(in string) string { - return strings.ToLower(tfVarRegex.ReplaceAllString(in, "")) -} - -// copyTfFiles - copies tf files that are related to current test -func copyTfFiles(testName string, tfTestFiles ...string) (string, error) { - workdir := filepath.Join(tfDir, testName) - if _, err := os.Stat(workdir); os.IsNotExist(err) { - if err := os.MkdirAll(workdir, os.ModePerm); err != nil { - return workdir, err - } - } else if err != nil { - return "", err - } - - files := make(map[string]string) - for _, tftf := range tfTestFiles { - files[filepath.Join(infraFilesDir, tftf)] = filepath.Join(workdir, tftf) - } - files[filepath.Join(infraFilesDir, "terraform.tf")] = filepath.Join(workdir, "terraform.tf") - files[filepath.Join(infraFilesDir, "provider.tf")] = filepath.Join(workdir, "provider.tf") - files[filepath.Join(infraFilesDir, "variables.tf")] = filepath.Join(workdir, "variables.tf") - - for src, dst := range files { - if _, err := os.Stat(src); err != nil { - return "", err - } - - in, err := os.ReadFile(src) - if err != nil { - return "", err - } - if err := os.WriteFile(dst, in, 0644); err != nil { - return "", err - } - } - return workdir, nil -} diff --git a/provider/testing/integration.md b/provider/testing/integration.md deleted file mode 100644 index a0723a36..00000000 --- a/provider/testing/integration.md +++ /dev/null @@ -1,38 +0,0 @@ -# Integration Tests - -## Description - -Integration tests use terraform to deploy resources. Every resource terraform file should be described in -./resources/testData and have the same name as tested resource(__.tf). -Example: `aws_iam_users.tf` -Testing routine copies this file among with default *.tf files to a separate folder, sets default variables prefix=< -resource_name> suffix=, deploys resources, fetches data from provider to database, queries data -using `Filter` field or using default filter based on tags, compares the received data with expected values. - -## Run - -To run integration tests you need: - -- see deployment examples in the cloudquery docs -- terraform executable in $PATH -- provider credentials configured via config files or env variables -- sql database deployed - -to run the tests use command below in PROVIDER root dir: -```shell - go test -v -p 20 ./resources --tags=integration -``` -Tests can be marked with `integration_skip` tag. This means are not ready yet for testing. - -## Debugging - -For debugging, you can set env variable `TF_NO_DESTROY=true` to leave the directory and resources after the test. -Resources should be destroyed manually by running - -``` -tf destroy -var test_prefix= -var test_suffix= -``` - -values for `-var` arguments can be found in test execution output near ` tf apply` log entry - -To avoid long terraform deploys you can set `TF_EXEC_TIMEOUT=` \ No newline at end of file diff --git a/provider/testing/resource.go b/provider/testing/resource.go index c5af4eb4..8d6f6782 100644 --- a/provider/testing/resource.go +++ b/provider/testing/resource.go @@ -2,135 +2,130 @@ package testing import ( "context" - "encoding/json" "fmt" "os" + "strings" + "sync" "testing" - "github.com/jackc/pgx/v4/pgxpool" - + sq "github.com/Masterminds/squirrel" "github.com/cloudquery/cq-provider-sdk/cqproto" - "github.com/cloudquery/cq-provider-sdk/logging" "github.com/cloudquery/cq-provider-sdk/provider" "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/testlog" "github.com/cloudquery/faker/v3" "github.com/georgysavva/scany/pgxscan" "github.com/hashicorp/go-hclog" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tmccombs/hcl2json/convert" ) -type ResourceTestData struct { +type ResourceTestCase struct { + Provider *provider.Provider Table *schema.Table - Config interface{} - Resources []string - Configure func(logger hclog.Logger, data interface{}) (schema.ClientMeta, error) + Config string + SnapshotsDir string SkipEmptyJsonB bool } -func TestResource(t *testing.T, providerCreator func() *provider.Provider, resource ResourceTestData) { +// IntegrationTest - creates resources using terraform, fetches them to db and compares with expected values +func TestResource(t *testing.T, resource ResourceTestCase) { + t.Parallel() + t.Helper() if err := faker.SetRandomMapAndSliceMinSize(1); err != nil { t.Fatal(err) } if err := faker.SetRandomMapAndSliceMaxSize(1); err != nil { t.Fatal(err) } - ctx := context.Background() + + // No need for configuration or db connection, get it out of the way first + // testTableIdentifiersForProvider(t, resource.Provider) pool, err := setupDatabase() if err != nil { t.Fatal(err) } - defer pool.Close() + ctx := context.Background() conn, err := pool.Acquire(ctx) if err != nil { t.Fatal(err) } defer conn.Release() - l := logging.New(hclog.DefaultOptions) - migrator := provider.NewTableCreator(l) - if err := migrator.CreateTable(ctx, conn, resource.Table, nil); err != nil { + + l := testlog.New(t) + l.SetLevel(hclog.Debug) + resource.Provider.Logger = l + tableCreator := provider.NewTableCreator(l) + if err := tableCreator.CreateTable(context.Background(), conn, resource.Table, nil); err != nil { assert.FailNow(t, fmt.Sprintf("failed to create tables %s", resource.Table.Name), err) } - // Write configuration as a block and extract it out passing that specific block data as part of the configure provider - f := hclwrite.NewFile() - f.Body().AppendBlock(gohcl.EncodeAsBlock(resource.Config, "configuration")) - data, err := convert.Bytes(f.Bytes(), "config.json", convert.Options{}) - require.Nil(t, err) - hack := map[string]interface{}{} - require.Nil(t, json.Unmarshal(data, &hack)) - data, err = json.Marshal(hack["configuration"].([]interface{})[0]) - require.Nil(t, err) - testProvider := providerCreator() + if err := deleteTables(conn, resource.Table); err != nil { + t.Fatal(err) + } - // No need for configuration or db connection, get it out of the way first - testTableIdentifiersForProvider(t, testProvider) + if err = fetch(t, &resource); err != nil { + t.Fatal(err) + } + + verifyNoEmptyColumns(t, resource, conn) + + if err := conn.Conn().Close(ctx); err != nil { + t.Fatal(err) + } + +} - testProvider.Logger = l - testProvider.Configure = resource.Configure - _, err = testProvider.ConfigureProvider(context.Background(), &cqproto.ConfigureProviderRequest{ +// fetch - fetches resources from the cloud and puts them into database. database config can be specified via DATABASE_URL env variable +func fetch(t *testing.T, resource *ResourceTestCase) error { + t.Logf("%s fetch resources", resource.Table.Name) + + if _, err := resource.Provider.ConfigureProvider(context.Background(), &cqproto.ConfigureProviderRequest{ CloudQueryVersion: "", Connection: cqproto.ConnectionDetails{DSN: getEnv("DATABASE_URL", "host=localhost user=postgres password=pass DB.name=postgres port=5432")}, - Config: data, - }) - assert.Nil(t, err) - - err = testProvider.FetchResources(context.Background(), &cqproto.FetchResourcesRequest{Resources: []string{findResourceFromTableName(resource.Table, testProvider.ResourceMap)}}, &fakeResourceSender{Errors: []string{}}) - assert.Nil(t, err) - verifyNoEmptyColumns(t, resource, conn) -} + Config: []byte(resource.Config), + DisableDelete: true, + }); err != nil { + return err + } -func findResourceFromTableName(table *schema.Table, tables map[string]*schema.Table) string { - for resource, t := range tables { - if table.Name == t.Name { - return resource - } + var resourceSender = &fakeResourceSender{ + Errors: []string{}, } - return "" -} -type fakeResourceSender struct { - Errors []string -} + if err := resource.Provider.FetchResources(context.Background(), + &cqproto.FetchResourcesRequest{ + Resources: []string{findResourceFromTableName(resource.Table, resource.Provider.ResourceMap)}, + }, + resourceSender, + ); err != nil { + return err + } -func (f *fakeResourceSender) Send(r *cqproto.FetchResourcesResponse) error { - if r.Error != "" { - fmt.Printf(r.Error) - f.Errors = append(f.Errors, r.Error) + if len(resourceSender.Errors) > 0 { + return fmt.Errorf("error/s occur during test, %s", strings.Join(resourceSender.Errors, ", ")) } + return nil } -func setupDatabase() (*pgxpool.Pool, error) { - dbCfg, err := pgxpool.ParseConfig(getEnv("DATABASE_URL", - "host=localhost user=postgres password=pass DB.name=postgres port=5432")) - if err != nil { - return nil, fmt.Errorf("failed to parse config. %w", err) - } - ctx := context.Background() - dbCfg.MaxConns = 1 - dbCfg.LazyConnect = true - pool, err := pgxpool.ConnectConfig(ctx, dbCfg) +func deleteTables(conn *pgxpool.Conn, table *schema.Table) error { + s := sq.Delete(table.Name) + sql, args, err := s.ToSql() if err != nil { - return nil, fmt.Errorf("unable to connect to database. %w", err) + return err } - return pool, nil - -} -func getEnv(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value + _, err = conn.Exec(context.TODO(), sql, args...) + if err != nil { + return err } - return fallback + return nil } -func verifyNoEmptyColumns(t *testing.T, tc ResourceTestData, conn pgxscan.Querier) { +func verifyNoEmptyColumns(t *testing.T, tc ResourceTestCase, conn pgxscan.Querier) { // Test that we don't have missing columns and have exactly one entry for each table for _, table := range getTablesFromMainTable(tc.Table) { query := fmt.Sprintf("select * FROM %s ", table) @@ -163,35 +158,61 @@ func verifyNoEmptyColumns(t *testing.T, tc ResourceTestData, conn pgxscan.Querie } } -func getTablesFromMainTable(table *schema.Table) []string { - var res []string - res = append(res, table.Name) - for _, t := range table.Relations { - res = append(res, getTablesFromMainTable(t)...) +func findResourceFromTableName(table *schema.Table, tables map[string]*schema.Table) string { + for resource, t := range tables { + if table.Name == t.Name { + return resource + } } - return res + return "" +} + +type fakeResourceSender struct { + Errors []string +} + +func (f *fakeResourceSender) Send(r *cqproto.FetchResourcesResponse) error { + if r.Error != "" { + fmt.Printf(r.Error) + f.Errors = append(f.Errors, r.Error) + } + return nil } -func testTableIdentifiersForProvider(t *testing.T, prov *provider.Provider) { - t.Run("testTableIdentifiersForProvider", func(t *testing.T) { - t.Parallel() - for _, res := range prov.ResourceMap { - res := res - t.Run(res.Name, func(t *testing.T) { - testTableIdentifiers(t, res) - }) +var ( + dbConnOnce sync.Once + pool *pgxpool.Pool + dbErr error +) + +func setupDatabase() (*pgxpool.Pool, error) { + dbConnOnce.Do(func() { + var dbCfg *pgxpool.Config + dbCfg, dbErr = pgxpool.ParseConfig(getEnv("DATABASE_URL", "host=localhost user=postgres password=pass DB.name=postgres port=5432")) + if dbErr != nil { + return } + ctx := context.Background() + dbCfg.MaxConns = 15 + dbCfg.LazyConnect = true + pool, dbErr = pgxpool.ConnectConfig(ctx, dbCfg) }) + return pool, dbErr + } -func testTableIdentifiers(t *testing.T, table *schema.Table) { - t.Parallel() - assert.NoError(t, schema.ValidateTable(table)) +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} - for _, res := range table.Relations { - res := res - t.Run(res.Name, func(t *testing.T) { - testTableIdentifiers(t, res) - }) +func getTablesFromMainTable(table *schema.Table) []string { + var res []string + res = append(res, table.Name) + for _, t := range table.Relations { + res = append(res, getTablesFromMainTable(t)...) } + return res } diff --git a/testlog/testlog.go b/testlog/testlog.go new file mode 100644 index 00000000..615eae74 --- /dev/null +++ b/testlog/testlog.go @@ -0,0 +1,137 @@ +package testlog + +import ( + "io" + "log" + "testing" + + "github.com/hashicorp/go-hclog" +) + +type testLogger struct { + t testing.TB + level hclog.Level +} + +// New create a new hclog adapter from testing.Log +// This is useful when running tests not to trash the output in tests +// and print only logs for tests that fail +func New(t testing.TB) hclog.Logger { + return &testLogger{ + t: t, + level: hclog.Debug, + } +} + +func (l *testLogger) Log(level hclog.Level, msg string, args ...interface{}) { + l.t.Helper() + switch level { + case hclog.NoLevel: + return + case hclog.Trace: + l.Trace(msg, args...) + case hclog.Debug: + l.Debug(msg, args...) + case hclog.Info: + l.Info(msg, args...) + case hclog.Warn: + l.Warn(msg, args...) + case hclog.Error: + l.Error(msg, args...) + } +} + +func (l *testLogger) Trace(msg string, args ...interface{}) { + l.t.Helper() + if l.level == hclog.Trace { + l.t.Log(convertMsgArgToInterface("[TRACE] "+msg, args)...) + } +} + +func (l *testLogger) Debug(msg string, args ...interface{}) { + l.t.Helper() + if l.IsDebug() { + l.t.Log(convertMsgArgToInterface("[DEBUG] "+msg, args)...) + } +} + +func (l *testLogger) Info(msg string, args ...interface{}) { + l.t.Helper() + if l.IsInfo() { + l.t.Log(convertMsgArgToInterface("[INFO] "+msg, args)...) + } +} + +func (l *testLogger) Warn(msg string, args ...interface{}) { + l.t.Helper() + if l.IsWarn() { + l.t.Log(convertMsgArgToInterface("[WARN] "+msg, args)...) + } +} + +func (l *testLogger) Error(msg string, args ...interface{}) { + l.t.Helper() + if l.IsError() { + l.t.Log(convertMsgArgToInterface("[ERROR] "+msg, args)...) + } +} + +func (l *testLogger) IsTrace() bool { + return l.level <= hclog.Trace +} + +func (l *testLogger) IsDebug() bool { + return l.level <= hclog.Debug +} + +func (l *testLogger) IsInfo() bool { + return l.level <= hclog.Info +} + +func (l *testLogger) IsWarn() bool { + return l.level <= hclog.Warn +} + +func (l *testLogger) IsError() bool { + return l.level <= hclog.Error +} + +// ImpliedArgs returns With key/value pairs +func (l *testLogger) ImpliedArgs() []interface{} { + return nil +} + +func (l *testLogger) With(args ...interface{}) hclog.Logger { + return l +} + +func (l *testLogger) Name() string { + return "testLogger" +} + +func (l *testLogger) Named(name string) hclog.Logger { + return l +} + +func (l *testLogger) ResetNamed(name string) hclog.Logger { + return l +} + +func (l *testLogger) SetLevel(level hclog.Level) { + l.level = level +} + +func (l *testLogger) StandardLogger(opts *hclog.StandardLoggerOptions) *log.Logger { + return nil +} + +func (l *testLogger) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer { + return nil +} + +func convertMsgArgToInterface(msg string, args ...interface{}) []interface{} { + var res []interface{} + res = append(res, msg) + res = append(res, args...) + return res +}