Implement minimal debug feature

cli/args.go

@@ -116,6 +116,8 @@ func parseTerragruntOptionsFromArgs(terragruntVersion string, args []string, wri
 
 	strictInclude := parseBooleanArg(args, OPT_TERRAGRUNT_STRICT_INCLUDE, false)
 
+	debug := parseBooleanArg(args, OPT_TERRAGRUNT_DEBUG, false)
+
 	opts, err := options.NewTerragruntOptions(filepath.ToSlash(terragruntConfigPath))
 	if err != nil {
 		return nil, err
@@ -161,6 +163,7 @@ func parseTerragruntOptionsFromArgs(terragruntVersion string, args []string, wri
 	opts.Parallelism = parallelism
 	opts.Check = parseBooleanArg(args, OPT_TERRAGRUNT_CHECK, os.Getenv("TERRAGRUNT_CHECK") == "true")
 	opts.HclFile = filepath.ToSlash(terragruntHclFilePath)
+	opts.Debug = debug
 
 	return opts, nil
 }

cli/args_test.go

@@ -35,67 +35,67 @@ func TestParseTerragruntOptionsFromArgs(t *testing.T) {
 	}{
 		{
 			[]string{},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"foo", "bar"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{"foo", "bar"}, false, "", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{"foo", "bar"}, false, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--foo", "--bar"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{"--foo", "--bar"}, false, "", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{"--foo", "--bar"}, false, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--foo", "apply", "--bar"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{"--foo", "apply", "--bar"}, false, "", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{"--foo", "apply", "--bar"}, false, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-non-interactive"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, true, "", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, true, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-include-external-dependencies"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, true),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, true, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-config", fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath)},
-			mockOptions(t, fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false),
+			mockOptions(t, fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-working-dir", "/some/path"},
-			mockOptions(t, util.JoinPath("/some/path", config.DefaultTerragruntConfigPath), "/some/path", []string{}, false, "", false, false),
+			mockOptions(t, util.JoinPath("/some/path", config.DefaultTerragruntConfigPath), "/some/path", []string{}, false, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-source", "/some/path"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "/some/path", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "/some/path", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-ignore-dependency-errors"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", true, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", true, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--terragrunt-ignore-external-dependencies"},
-			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false),
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false, false),
 			nil,
 		},
 
@@ -107,13 +107,20 @@ func TestParseTerragruntOptionsFromArgs(t *testing.T) {
 
 		{
 			[]string{"--terragrunt-config", fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), "--terragrunt-non-interactive"},
-			mockOptions(t, fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), workingDir, []string{}, true, "", false, false),
+			mockOptions(t, fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), workingDir, []string{}, true, "", false, false, false),
 			nil,
 		},
 
 		{
 			[]string{"--foo", "--terragrunt-config", fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), "bar", "--terragrunt-non-interactive", "--baz", "--terragrunt-working-dir", "/some/path", "--terragrunt-source", "github.com/foo/bar//baz?ref=1.0.3"},
-			mockOptions(t, fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), "/some/path", []string{"--foo", "bar", "--baz"}, true, "github.com/foo/bar//baz?ref=1.0.3", false, false),
+			mockOptions(t, fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath), "/some/path", []string{"--foo", "bar", "--baz"}, true, "github.com/foo/bar//baz?ref=1.0.3", false, false, false),
+			nil,
+		},
+
+		// Adding the --terragrunt-debug flag should result in an Options with Debug set to true
+		{
+			[]string{"--terragrunt-debug"},
+			mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, []string{}, false, "", false, false, true),
 			nil,
 		},
 
@@ -161,9 +168,10 @@ func assertOptionsEqual(t *testing.T, expected options.TerragruntOptions, actual
 	assert.Equal(t, expected.Source, actual.Source, msgAndArgs...)
 	assert.Equal(t, expected.IgnoreDependencyErrors, actual.IgnoreDependencyErrors, msgAndArgs...)
 	assert.Equal(t, expected.IamRole, actual.IamRole, msgAndArgs...)
+	assert.Equal(t, expected.Debug, actual.Debug, msgAndArgs...)
 }
 
-func mockOptions(t *testing.T, terragruntConfigPath string, workingDir string, terraformCliArgs []string, nonInteractive bool, terragruntSource string, ignoreDependencyErrors bool, includeExternalDependencies bool) *options.TerragruntOptions {
+func mockOptions(t *testing.T, terragruntConfigPath string, workingDir string, terraformCliArgs []string, nonInteractive bool, terragruntSource string, ignoreDependencyErrors bool, includeExternalDependencies bool, debugMode bool) *options.TerragruntOptions {
 	opts, err := options.NewTerragruntOptionsForTest(terragruntConfigPath)
 	if err != nil {
 		t.Fatalf("error: %v\n", errors.WithStackTrace(err))
@@ -175,12 +183,13 @@ func mockOptions(t *testing.T, terragruntConfigPath string, workingDir string, t
 	opts.Source = terragruntSource
 	opts.IgnoreDependencyErrors = ignoreDependencyErrors
 	opts.IncludeExternalDependencies = includeExternalDependencies
+	opts.Debug = debugMode
 
 	return opts
 }
 
 func mockOptionsWithIamRole(t *testing.T, terragruntConfigPath string, workingDir string, terraformCliArgs []string, nonInteractive bool, terragruntSource string, ignoreDependencyErrors bool, iamRole string) *options.TerragruntOptions {
-	opts := mockOptions(t, terragruntConfigPath, workingDir, terraformCliArgs, nonInteractive, terragruntSource, ignoreDependencyErrors, false)
+	opts := mockOptions(t, terragruntConfigPath, workingDir, terraformCliArgs, nonInteractive, terragruntSource, ignoreDependencyErrors, false, false)
 	opts.IamRole = iamRole
 
 	return opts
@@ -197,6 +206,7 @@ func TestFilterTerragruntArgs(t *testing.T) {
 		{[]string{"foo", "--bar"}, []string{"foo", "--bar"}},
 		{[]string{"foo", "--terragrunt-config", fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath)}, []string{"foo"}},
 		{[]string{"foo", "--terragrunt-non-interactive"}, []string{"foo"}},
+		{[]string{"foo", "--terragrunt-debug"}, []string{"foo"}},
 		{[]string{"foo", "--terragrunt-non-interactive", "--bar", "--terragrunt-working-dir", "/some/path", "--baz", "--terragrunt-config", fmt.Sprintf("/some/path/%s", config.DefaultTerragruntConfigPath)}, []string{"foo", "--bar", "--baz"}},
 		{[]string{"apply-all", "foo", "bar"}, []string{"foo", "bar"}},
 		{[]string{"foo", "destroy-all", "--foo", "--bar"}, []string{"foo", "--foo", "--bar"}},
@@ -524,7 +534,7 @@ func createTempFile(t *testing.T) string {
 }
 
 func mockCmdOptions(t *testing.T, workingDir string, terraformCliArgs []string) *options.TerragruntOptions {
-	o := mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, terraformCliArgs, true, "", false, false)
+	o := mockOptions(t, util.JoinPath(workingDir, config.DefaultTerragruntConfigPath), workingDir, terraformCliArgs, true, "", false, false, false)
 	return o
 }
 

cli/cli_app.go

@@ -42,6 +42,7 @@ const OPT_TERRAGRUNT_STRICT_INCLUDE = "terragrunt-strict-include"
 const OPT_TERRAGRUNT_PARALLELISM = "terragrunt-parallelism"
 const OPT_TERRAGRUNT_CHECK = "terragrunt-check"
 const OPT_TERRAGRUNT_HCLFMT_FILE = "terragrunt-hclfmt-file"
+const OPT_TERRAGRUNT_DEBUG = "terragrunt-debug"
 
 var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{
 	OPT_NON_INTERACTIVE,
@@ -54,6 +55,7 @@ var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{
 	OPT_TERRAGRUNT_NO_AUTO_RETRY,
 	OPT_TERRAGRUNT_CHECK,
 	OPT_TERRAGRUNT_STRICT_INCLUDE,
+	OPT_TERRAGRUNT_DEBUG,
 }
 var ALL_TERRAGRUNT_STRING_OPTS = []string{
 	OPT_TERRAGRUNT_CONFIG,
@@ -174,6 +176,7 @@ GLOBAL OPTIONS:
    terragrunt-include-dir                       Unix-style glob of directories to include when running *-all commands
    terragrunt-check                             Enable check mode in the hclfmt command.
    terragrunt-hclfmt-file                       The path to a single terragrunt.hcl file that the hclfmt command should run on.
+   terragrunt-debug                             Write terragrunt-debug.tfvars to working folder to help root-cause issues.
 
 VERSION:
    {{.Version}}{{if len .Authors}}
@@ -325,6 +328,8 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 		}
 	}
 
+	// NOTE: At this point, the terraform source is downloaded to the terragrunt working directory
+
 	if shouldPrintTerragruntInfo(terragruntOptions) {
 		group := TerragruntInfoGroup{
 			ConfigPath:       terragruntOptions.TerragruntConfigPath,
@@ -366,6 +371,15 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 		}
 	}
 
+	// We do the debug file generation here, after all the terragrunt generated terraform files are created so that we
+	// can ensure the tfvars json file only includes the vars that are defined in the module.
+	if terragruntOptions.Debug {
+		err := writeTerragruntDebugFile(terragruntOptions, terragruntConfig)
+		if err != nil {
+			return err
+		}
+	}
+
 	return runTerragruntWithConfig(terragruntOptions, terragruntConfig, false)
 }
 
@@ -576,7 +590,6 @@ func setTerragruntInputsAsEnvVars(terragruntOptions *options.TerragruntOptions,
 			terragruntOptions.Env[key] = value
 		}
 	}
-
 	return nil
 }
 

cli/debug.go

@@ -0,0 +1,118 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/terraform-config-inspect/tfconfig"
+
+	"github.com/gruntwork-io/terragrunt/config"
+	"github.com/gruntwork-io/terragrunt/errors"
+	"github.com/gruntwork-io/terragrunt/options"
+	"github.com/gruntwork-io/terragrunt/util"
+)
+
+const TerragruntTFVarsFile = "terragrunt-debug.tfvars.json"
+
+// writeTerragruntDebugFile will create a tfvars file that can be used to invoke the terraform module in the same way
+// that terragrunt invokes the module, so that you can debug issues with the terragrunt config.
+func writeTerragruntDebugFile(terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) error {
+	terragruntOptions.Logger.Printf(
+		"Debug mode requested: generating debug file %s in working dir %s",
+		TerragruntTFVarsFile,
+		terragruntOptions.WorkingDir,
+	)
+
+	variables, err := terraformModuleVariables(terragruntOptions)
+	if err != nil {
+		return err
+	}
+	util.Debugf(terragruntOptions.Logger, "The following variables were detected in the terraform module:")
+	util.Debugf(terragruntOptions.Logger, "%v", variables)
+
+	fileContents, err := terragruntDebugFileContents(terragruntOptions, terragruntConfig, variables)
+	if err != nil {
+		return err
+	}
+
+	configFolder := filepath.Dir(terragruntOptions.TerragruntConfigPath)
+	fileName := filepath.Join(configFolder, TerragruntTFVarsFile)
+	if err := ioutil.WriteFile(fileName, fileContents, os.FileMode(int(0600))); err != nil {
+		return errors.WithStackTrace(err)
+	}
+
+	terragruntOptions.Logger.Printf("Variables passed to terraform are located in \"%s\"", fileName)
+	terragruntOptions.Logger.Printf("Run this command to replicate how terraform was invoked:")
+	terragruntOptions.Logger.Printf(
+		"\tterraform %s -var-file=\"%s\" \"%s\"",
+		strings.Join(terragruntOptions.TerraformCliArgs, " "),
+		fileName,
+		terragruntOptions.WorkingDir,
+	)
+	return nil
+}
+
+// terragruntDebugFileContents will return a tfvars file in json format of all the terragrunt rendered variables values
+// that should be set to invoke the terraform module in the same way as terragrunt. Note that this will only include the
+// values of variables that are actually defined in the module.
+func terragruntDebugFileContents(
+	terragruntOptions *options.TerragruntOptions,
+	terragruntConfig *config.TerragruntConfig,
+	moduleVariables []string,
+) ([]byte, error) {
+	envVars := map[string]string{}
+	if terragruntOptions.Env != nil {
+		envVars = terragruntOptions.Env
+	}
+
+	jsonValuesByKey := make(map[string]interface{})
+	for varName, varValue := range terragruntConfig.Inputs {
+		nameAsEnvVar := fmt.Sprintf("TF_VAR_%s", varName)
+		_, varIsInEnv := envVars[nameAsEnvVar]
+		varIsDefined := util.ListContainsElement(moduleVariables, varName)
+
+		// Only add to the file if the explicit env var does NOT exist and the variable is defined in the module.
+		// We must do this in order to avoid overriding the env var when the user follows up with a direct invocation to
+		// terraform using this file (due to the order in which terraform resolves config sources).
+		if !varIsInEnv && varIsDefined {
+			jsonValuesByKey[varName] = varValue
+		} else if varIsInEnv {
+			util.Debugf(
+				terragruntOptions.Logger,
+				"WARN: The variable %s was omitted from the debug file because the env var %s is already set.",
+				varName, nameAsEnvVar,
+			)
+		} else if !varIsDefined {
+			util.Debugf(
+				terragruntOptions.Logger,
+				"WARN: The variable %s was omitted because it is not defined in the terraform module.",
+				varName,
+			)
+		}
+	}
+	jsonContent, err := json.MarshalIndent(jsonValuesByKey, "", "  ")
+	if err != nil {
+		return nil, errors.WithStackTrace(err)
+	}
+	return jsonContent, nil
+}
+
+// terraformModuleVariables will return all the variables defined in the downloaded terraform modules, taking into
+// account all the generated sources.
+func terraformModuleVariables(terragruntOptions *options.TerragruntOptions) ([]string, error) {
+	modulePath := terragruntOptions.WorkingDir
+	module, diags := tfconfig.LoadModule(modulePath)
+	if diags.HasErrors() {
+		return nil, errors.WithStackTrace(diags)
+	}
+
+	variables := []string{}
+	for _, variable := range module.Variables {
+		variables = append(variables, variable.Name)
+	}
+	return variables, nil
+}