Make temp directory cleanup easier to control

- Maintain existing defaults - keeping locally.
- Add explicit control via PULUMITEST_RETAIN_FILES_ON_FAILURE with prompting to the user on failure.
- Default to retaining if we've retained the resources.
- Finally, check for the `CI` env var.

pulumitest/cleanup.go

@@ -5,6 +5,9 @@ import (
 	"strings"
 )
 
+// We use this interesting contraption of immediately invoked callbacks to
+// eagerly calculate these variables but ensure they're not mutated.
+
 var skipDestroyOnFailure = (func() func() bool {
 	value, ok := os.LookupEnv("PULUMITEST_SKIP_DESTROY_ON_FAILURE")
 	skipDestroy := ok && strings.EqualFold(value, "true")
@@ -15,3 +18,16 @@ var runningInCI = (func() func() bool {
 	_, ok := os.LookupEnv("CI")
 	return func() bool { return ok }
 })()
+
+var shouldRetainFilesOnFailure = (func() func() bool {
+	if value, ok := os.LookupEnv("PULUMITEST_RETAIN_FILES_ON_FAILURE"); ok {
+		if strings.EqualFold(value, "false") {
+			return func() bool { return false }
+		}
+		return func() bool { return true }
+	}
+	if skipDestroyOnFailure() {
+		return func() bool { return true }
+	}
+	return func() bool { return !runningInCI() }
+})()

pulumitest/tempdir.go

@@ -53,9 +53,14 @@ func tempDirWithoutCleanupOnFailedTest(t PT, desc string) string {
 		if c.tempDirErr == nil {
 			t.Cleanup(func() {
 				t.Helper()
-				if ptFailed(t) && !runningInCI() {
-					ptErrorF(t, "TempDir leaving %s to help debugging: %q", desc, c.tempDir)
-				} else if err := os.RemoveAll(c.tempDir); err != nil {
+				if ptFailed(t) && shouldRetainFilesOnFailure() {
+					ptLogF(t, "Skipping removal of %s temp directories on failures: %q", desc, c.tempDir)
+					t.Log("To remove these directories on failures, set PULUMITEST_RETAIN_FILES_ON_FAILURE=false")
+					return
+				}
+				err := os.RemoveAll(c.tempDir)
+				t.Log("Removed temp directories. To retain these, set PULUMITEST_RETAIN_FILES_ON_FAILURE=true")
+				if err != nil {
 					ptErrorF(t, "TempDir RemoveAll cleanup: %v", err)
 				}
 			})