refactor: e2e test use conftest command instead command$version (#2851)

server/controllers/events/events_controller_e2e_test.go

@@ -42,7 +42,9 @@ import (
 	. "github.com/runatlantis/atlantis/testing"
 )
 
-const ConftestVersion = "0.36.0" // renovate: datasource=github-releases depName=open-policy-agent/conftest
+// In the e2e test, we use `conftest` not `conftest$version`.
+// Because if depends on the version, we need to upgrade test base image before e2e fix it.
+const conftestCommand = "conftest"
 
 var applyLocker locking.ApplyLocker
 var userConfig server.UserConfig
@@ -64,8 +66,8 @@ func (m *NoopTFDownloader) GetAny(dst, src string, opts ...getter.ClientOption)
 type LocalConftestCache struct {
 }
 
-func (m *LocalConftestCache) Get(key *version.Version) (string, error) {
-	return exec.LookPath(fmt.Sprintf("conftest%s", ConftestVersion))
+func (m *LocalConftestCache) Get(_ *version.Version) (string, error) {
+	return exec.LookPath(conftestCommand)
 }
 
 func TestGitHubWorkflow(t *testing.T) {
@@ -886,13 +888,6 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers.
 	allowForkPRs := false
 	dataDir, binDir, cacheDir := mkSubDirs(t)
 
-	//env vars
-
-	if userConfig.EnablePolicyChecksFlag {
-		// need this to be set or we'll fail the policy check step
-		os.Setenv(policy.DefaultConftestVersionEnvKey, "0.25.0")
-	}
-
 	// Mocks.
 	e2eVCSClient := vcsmocks.NewMockClient()
 	e2eStatusUpdater := &events.DefaultCommitStatusUpdater{Client: e2eVCSClient}
@@ -1016,16 +1011,14 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers.
 
 	Ok(t, err)
 
-	conftestVersion, _ := version.NewVersion(ConftestVersion)
-
 	conftextExec := policy.NewConfTestExecutorWorkflow(logger, binDir, &NoopTFDownloader{})
 
 	// swapping out version cache to something that always returns local contest
 	// binary
 	conftextExec.VersionCache = &LocalConftestCache{}
 
 	policyCheckRunner, err := runtime.NewPolicyCheckStepRunner(
-		conftestVersion,
+		defaultTFVersion,
 		conftextExec,
 	)
 
@@ -1392,30 +1385,16 @@ func mkSubDirs(t *testing.T) (string, string, string) {
 	return tmp, binDir, cachedir
 }
 
-// Will fail test if conftest isn't in path or is version less than specific version
+// Will fail test if conftest isn't in path
 func ensureRunningConftest(t *testing.T) {
-	localPath, err := exec.LookPath("conftest")
+	// use `conftest` command instead `contest$version`, so tests may fail on the environment cause the output logs may become change by version.
+	t.Logf("conftest check may fail depends on conftest version. please use latest stable conftest.")
+	_, err := exec.LookPath(conftestCommand)
 	if err != nil {
-		t.Logf("conftest must be installed to run this test")
-		t.FailNow()
-	}
-	versionOutBytes, err := exec.Command(localPath, "--version").Output() // #nosec
-	if err != nil {
-		t.Logf("error running conftest version: %s", err)
-		t.FailNow()
-	}
-	versionOutput := string(versionOutBytes)
-	match := versionConftestRegex.FindStringSubmatch(versionOutput)
-	if len(match) <= 1 {
-		t.Logf("could not parse conftest version from %s", versionOutput)
-		t.FailNow()
-	}
-	localVersion, err := version.NewVersion(match[1])
-	Ok(t, err)
-	minVersion, err := version.NewVersion(ConftestVersion)
-	Ok(t, err)
-	if localVersion.LessThan(minVersion) {
-		t.Logf("must have contest version >= %s, you have %s", minVersion, localVersion)
+		t.Logf(`%s must be installed to run this test
+- on local, please install contest command or run 'make docker/test-all'
+- on CI, please check testing-env docker image contains conftest command. see testing/Dockerfile
+`, conftestCommand)
 		t.FailNow()
 	}
 }
@@ -1456,10 +1435,3 @@ func ensureRunning014(t *testing.T) {
 //	    Terraform v0.11.10
 //		   => 0.11.10
 var versionRegex = regexp.MustCompile("Terraform v(.*?)(\\s.*)?\n")
-
-/*
- * Newer versions will return both Conftest and OPA
- * Conftest: 0.35.0
- * OPA: 0.45.0
- */
-var versionConftestRegex = regexp.MustCompile("Conftest: (.*?)(\\s.*)?\n")

server/core/runtime/policy/conftest_client.go

@@ -211,9 +211,14 @@ func (c *ConfTestExecutorWorkflow) sanitizeOutput(inputFile string, output strin
 }
 
 func (c *ConfTestExecutorWorkflow) EnsureExecutorVersion(log logging.SimpleLogging, v *version.Version) (string, error) {
-	// we have no information to proceed so fail hard
+	// we have no information to proceed, so fallback to `conftest` command or fail hard
 	if c.DefaultConftestVersion == nil && v == nil {
-		return "", errors.New("no conftest version configured/specified")
+		localPath, err := c.Exec.LookPath(conftestBinaryName)
+		if err == nil {
+			log.Info("conftest version is not specified, so fallback to conftest command")
+			return localPath, nil
+		}
+		return "", errors.New("no conftest version configured/specified or not found conftest command")
 	}
 
 	var versionToRetrieve *version.Version

server/core/runtime/policy/conftest_client_test.go

@@ -64,18 +64,32 @@ func TestEnsureExecutorVersion(t *testing.T) {
 	RegisterMockTestingT(t)
 
 	mockCache := mocks.NewMockExecutionVersionCache()
+	mockExec := models_mocks.NewMockExec()
 	log := logging.NewNoopLogger(t)
 
-	t.Run("no specified version or default version", func(t *testing.T) {
+	t.Run("no specified version or default version without conftest command", func(t *testing.T) {
 		subject := &ConfTestExecutorWorkflow{
 			VersionCache: mockCache,
+			Exec:         mockExec,
 		}
 
+		When(mockExec.LookPath(AnyString())).ThenReturn("", errors.New("not found"))
 		_, err := subject.EnsureExecutorVersion(log, nil)
 
 		Assert(t, err != nil, "expected error finding version")
 	})
 
+	t.Run("no specified version or default version with conftest command", func(t *testing.T) {
+		subject := &ConfTestExecutorWorkflow{
+			VersionCache: mockCache,
+			Exec:         mockExec,
+		}
+		When(mockExec.LookPath(AnyString())).ThenReturn(expectedPath, nil)
+		path, err := subject.EnsureExecutorVersion(log, nil)
+		Ok(t, err)
+		Assert(t, path == expectedPath, "path is expected")
+	})
+
 	t.Run("use default version", func(t *testing.T) {
 		subject := &ConfTestExecutorWorkflow{
 			VersionCache:           mockCache,

testing/Dockerfile

@@ -24,8 +24,8 @@ RUN case $(uname -m) in x86_64|amd64) ARCH="x86_64" ;; aarch64|arm64|armv7l) ARC
     sed -n "/conftest_${CONFTEST_VERSION}_Linux_${ARCH}.tar.gz/p" checksums.txt | sha256sum -c && \
     mkdir -p /usr/local/bin/cft/versions/${CONFTEST_VERSION} && \
     tar -C  /usr/local/bin/cft/versions/${CONFTEST_VERSION} -xzf conftest_${CONFTEST_VERSION}_Linux_${ARCH}.tar.gz && \
-    # After migrating e2e test which use contest command instead of contest$version, you can remove contest$version command.
-    ln -s /usr/local/bin/cft/versions/${CONFTEST_VERSION}/conftest /usr/local/bin/conftest${CONFTEST_VERSION} && \
+    # Generally Atlantis requires `conftest$version` command. But we use `conftest` command in test.
+    # `conftest$version` command blocks upgrading conftest operation cause e2e test use this image.
     ln -s /usr/local/bin/cft/versions/${CONFTEST_VERSION}/conftest /usr/local/bin/conftest && \
     rm conftest_${CONFTEST_VERSION}_Linux_${ARCH}.tar.gz && \
     rm checksums.txt