Add better version matrix handling and additional version testing for all e2e tests

.circleci/config.yml

@@ -30,26 +30,27 @@ jobs:
       - win_install_go
       - go_test
 
-  "go112_build":
+  go112_build:
     docker:
       - image: circleci/golang:1.12
     steps:
       - checkout
       - go_build
-  "go112_test":
+
+  go113_build:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     steps:
       - checkout
       - go_build
 
-  "go114_build":
+  go114_build:
     docker:
       - image: circleci/golang:1.14
     steps:
       - checkout
       - go_build
-  "go114_test":
+  go114_test:
     docker:
       - image: circleci/golang:1.14
     parameters:
@@ -59,25 +60,42 @@ jobs:
     steps:
       - checkout
       - go_test
-  "go114_vet":
+
+  go115_build:
     docker:
-      - image: circleci/golang:1.14
+      - image: circleci/golang:1.15
+    steps:
+      - checkout
+      - go_build
+  go115_test:
+    docker:
+      - image: circleci/golang:1.15
+    parameters:
+      test_results:
+        type: string
+        default: /tmp/test-results
+    steps:
+      - checkout
+      - go_test
+  go115_vet:
+    docker:
+      - image: circleci/golang:1.15
     steps:
       - checkout
       - run: go vet ./...
-  "go114_fmt":
+  go115_fmt:
     docker:
-      - image: circleci/golang:1.14
+      - image: circleci/golang:1.15
     steps:
       - checkout
       - run: gofmt -s -l .
-  "go114_release":
+  go115_release:
     docker:
-      - image: circleci/golang:1.14
+      - image: circleci/golang:1.15
     steps:
       - add_ssh_keys:
           fingerprints:
-            - "db:cf:97:b8:d6:ac:86:74:96:e1:54:e4:bc:27:2b:d0"
+            - db:cf:97:b8:d6:ac:86:74:96:e1:54:e4:bc:27:2b:d0
       - checkout
       - run: ./scripts/release/release.sh
 
@@ -86,50 +104,74 @@ workflows:
   pr:
     jobs:
       - winbuild
-      - wintest
-      - "go112_build"
-      - "go112_test"
-      - "go114_build"
-      - "go114_test":
+      - wintest:
+          requires:
+            - winbuild
+
+      # build only for these versions
+      - go112_build
+      - go113_build
+
+      - go114_build
+      - go114_test:
+          requires:
+            - go114_build
+
+      - go115_build
+      - go115_test:
           requires:
-            - "go114_build"
-      - "go114_vet":
+            - go115_build
+      - go115_vet:
           requires:
-            - "go114_build"
-      - "go114_fmt":
+            - go115_build
+      - go115_fmt:
           requires:
-            - "go114_build"
+            - go115_build
   release:
     jobs:
       - winbuild
-      - wintest
-      - "go112_build"
-      - "go112_test"
-      - "go114_build"
-      - "go114_test":
+      - wintest:
+          requires:
+            - winbuild
+
+      # build only for these versions
+      - go112_build
+      - go113_build
+
+      - go114_build
+      - go114_test:
+          requires:
+            - go114_build
+
+      - go115_build
+      - go115_test:
           requires:
-            - "go114_build"
-      - "go114_vet":
+            - go115_build
+      - go115_vet:
           requires:
-            - "go114_build"
-      - "go114_fmt":
+            - go115_build
+      - go115_fmt:
           requires:
-            - "go114_build"
+            - go115_build
+
       - trigger-release:
           filters:
             branches:
               only:
                 - master
           type: approval
-      - "go114_release":
+
+      - go115_release:
           filters:
             branches:
               only:
                 - master
           requires:
             - trigger-release
-            - "go114_test"
-            - "go114_vet"
-            - "go114_fmt"
-            - winbuild
+            - go112_build
+            - go113_build
+            - go114_test
+            - go115_test
+            - go115_vet
+            - go115_fmt
             - wintest

tfexec/errors.go

@@ -19,6 +19,9 @@ func parseError(err error, stderr string) error {
 		return &ErrCLIUsage{stderr: stderr}
 	case regexp.MustCompile(`Error: Could not satisfy plugin requirements`).MatchString(stderr):
 		return &ErrNoInit{stderr: stderr}
+	case regexp.MustCompile(`Error: Could not load plugin`).MatchString(stderr):
+		// this string is present in 0.13
+		return &ErrNoInit{stderr: stderr}
 	case regexp.MustCompile(`Error: No configuration files`).MatchString(stderr):
 		return &ErrNoConfig{stderr: stderr}
 	default:

tfexec/internal/e2etest/apply_test.go

@@ -4,20 +4,21 @@ import (
 	"context"
 	"testing"
 
-	"github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+	"github.com/hashicorp/go-version"
+
+	"github.com/hashicorp/terraform-exec/tfexec"
 )
 
 func TestApply(t *testing.T) {
-	tf, cleanup := setupFixture(t, testutil.Latest012, "basic")
-	defer cleanup()
-
-	err := tf.Init(context.Background())
-	if err != nil {
-		t.Fatalf("error running Init in test directory: %s", err)
-	}
+	runTest(t, "basic", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+		err := tf.Init(context.Background())
+		if err != nil {
+			t.Fatalf("error running Init in test directory: %s", err)
+		}
 
-	err = tf.Apply(context.Background())
-	if err != nil {
-		t.Fatalf("error running Apply: %s", err)
-	}
+		err = tf.Apply(context.Background())
+		if err != nil {
+			t.Fatalf("error running Apply: %s", err)
+		}
+	})
 }

tfexec/internal/e2etest/destroy_test.go

@@ -4,25 +4,26 @@ import (
 	"context"
 	"testing"
 
-	"github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+	"github.com/hashicorp/go-version"
+
+	"github.com/hashicorp/terraform-exec/tfexec"
 )
 
 func TestDestroy(t *testing.T) {
-	tf, cleanup := setupFixture(t, testutil.Latest012, "basic")
-	defer cleanup()
-
-	err := tf.Init(context.Background())
-	if err != nil {
-		t.Fatalf("error running Init in test directory: %s", err)
-	}
+	runTest(t, "basic", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+		err := tf.Init(context.Background())
+		if err != nil {
+			t.Fatalf("error running Init in test directory: %s", err)
+		}
 
-	err = tf.Apply(context.Background())
-	if err != nil {
-		t.Fatalf("error running Apply: %s", err)
-	}
+		err = tf.Apply(context.Background())
+		if err != nil {
+			t.Fatalf("error running Apply: %s", err)
+		}
 
-	err = tf.Destroy(context.Background())
-	if err != nil {
-		t.Fatalf("error running Destroy: %s", err)
-	}
+		err = tf.Destroy(context.Background())
+		if err != nil {
+			t.Fatalf("error running Destroy: %s", err)
+		}
+	})
 }

tfexec/internal/e2etest/errors_test.go

@@ -9,35 +9,29 @@ import (
 	"os"
 	"testing"
 
-	"github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+	"github.com/hashicorp/go-version"
+
+	"github.com/hashicorp/terraform-exec/tfexec"
 )
 
 func TestUnparsedError(t *testing.T) {
 	// This simulates an unparsed error from the Cmd.Run method (in this case file not found). This
 	// is to ensure we don't miss raising unexpected errors in addition to parsed / well known ones.
-	for _, tfv := range []string{
-		testutil.Latest011,
-		testutil.Latest012,
-		testutil.Latest013,
-	} {
-		t.Run(tfv, func(t *testing.T) {
-			tf, cleanup := setupFixture(t, tfv, "")
-			defer cleanup()
+	runTest(t, "", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
 
-			// force delete the working dir to cause an os.PathError
-			err := os.RemoveAll(tf.WorkingDir())
-			if err != nil {
-				t.Fatal(err)
-			}
+		// force delete the working dir to cause an os.PathError
+		err := os.RemoveAll(tf.WorkingDir())
+		if err != nil {
+			t.Fatal(err)
+		}
 
-			err = tf.Init(context.Background())
-			if err == nil {
-				t.Fatalf("expected error running Init, none returned")
-			}
-			var e *os.PathError
-			if !errors.As(err, &e) {
-				t.Fatalf("expected os.PathError, got %T, %s", err, err)
-			}
-		})
-	}
+		err = tf.Init(context.Background())
+		if err == nil {
+			t.Fatalf("expected error running Init, none returned")
+		}
+		var e *os.PathError
+		if !errors.As(err, &e) {
+			t.Fatalf("expected os.PathError, got %T, %s", err, err)
+		}
+	})
 }