Dag for local data eval

command/build.go

@@ -102,7 +102,9 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 		return ret
 	}
 
-	diags = packerStarter.Initialize(packer.InitializeOptions{})
+	diags = packerStarter.Initialize(packer.InitializeOptions{
+		UseSequential: cla.UseSequential,
+	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret
@@ -439,6 +441,7 @@ Options:
   -var-file=path                JSON or HCL2 file containing user variables, can be used multiple times.
   -warn-on-undeclared-var       Display warnings for user variable files containing undeclared variables.
   -ignore-prerelease-plugins    Disable the loading of prerelease plugin binaries (x.y.z-dev).
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

command/cli.go

@@ -75,6 +75,12 @@ type MetaArgs struct {
 	// WarnOnUndeclared does not have a common default, as the default varies per sub-command usage.
 	// Refer to individual command FlagSets for usage.
 	WarnOnUndeclaredVar bool
+	// UseSequential specifies to use a sequential/phased approach for
+	// evaluating datasources/locals instead of a DAG.
+	//
+	// This allows users to fall-back to using the approach used by Packer
+	// before the introduction of a DAG in case they run in an impasse/bug.
+	UseSequential bool
 }
 
 func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
@@ -90,6 +96,7 @@ func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.Var(flagOnError, "on-error", "")
 
 	flags.BoolVar(&ba.MetaArgs.WarnOnUndeclaredVar, "warn-on-undeclared-var", false, "Show warnings for variable files containing undeclared variables.")
+	flags.BoolVar(&ba.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 
 	flags.BoolVar(&ba.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).")
 
@@ -156,6 +163,7 @@ type ConsoleArgs struct {
 
 func (fa *FixArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.BoolVar(&fa.Validate, "validate", true, "")
+	flags.BoolVar(&fa.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 
 	fa.MetaArgs.AddFlagSets(flags)
 }
@@ -171,6 +179,7 @@ func (va *ValidateArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.BoolVar(&va.NoWarnUndeclaredVar, "no-warn-undeclared-var", false, "Ignore warnings for variable files containing undeclared variables.")
 	flags.BoolVar(&va.EvaluateDatasources, "evaluate-datasources", false, "evaluate datasources for validation (HCL2 only, may incur costs)")
 	flags.BoolVar(&va.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).")
+	flags.BoolVar(&va.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 
 	va.MetaArgs.AddFlagSets(flags)
 }
@@ -184,6 +193,7 @@ type ValidateArgs struct {
 }
 
 func (va *InspectArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&va.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 	va.MetaArgs.AddFlagSets(flags)
 }
 

command/console.go

@@ -63,7 +63,9 @@ func (c *ConsoleCommand) RunContext(ctx context.Context, cla *ConsoleArgs) int {
 		return ret
 	}
 
-	_ = packerStarter.Initialize(packer.InitializeOptions{})
+	_ = packerStarter.Initialize(packer.InitializeOptions{
+		UseSequential: cla.UseSequential,
+	})
 
 	// Determine if stdin is a pipe. If so, we evaluate directly.
 	if c.StdinPiped() {
@@ -83,9 +85,10 @@ Usage: packer console [options] [TEMPLATE]
   interpolation.
 
 Options:
-  -var 'key=value'       Variable for templates, can be used multiple times.
-  -var-file=path         JSON or HCL2 file containing user variables.
-  -config-type           Set to 'hcl2' to run in HCL2 mode when no file is passed. Defaults to json.
+  -var 'key=value'              Variable for templates, can be used multiple times.
+  -var-file=path                JSON or HCL2 file containing user variables.
+  -config-type                  Set to 'hcl2' to run in HCL2 mode when no file is passed. Defaults to json.
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

command/hcl2_upgrade.go

@@ -181,7 +181,12 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs)
 	}
 
 	core := hdl.(*packer.Core)
-	if err := core.Initialize(packer.InitializeOptions{}); err != nil {
+	if err := core.Initialize(packer.InitializeOptions{
+		// Note: this is always true here as the DAG is only usable for
+		// HCL2 configs, so since the command only works on JSON templates,
+		// we can safely use the phased approach, which changes nothing.
+		UseSequential: true,
+	}); err != nil {
 		c.Ui.Error(fmt.Sprintf("Ignoring following initialization error: %v", err))
 	}
 	tpl := core.Template

command/inspect.go

@@ -49,7 +49,9 @@ func (c *InspectCommand) RunContext(ctx context.Context, cla *InspectArgs) int {
 	}
 
 	// here we ignore init diags to allow unknown variables to be used
-	_ = packerStarter.Initialize(packer.InitializeOptions{})
+	_ = packerStarter.Initialize(packer.InitializeOptions{
+		UseSequential: cla.UseSequential,
+	})
 
 	return packerStarter.InspectConfig(packer.InspectConfigOptions{
 		Ui: c.Ui,
@@ -66,7 +68,8 @@ Usage: packer inspect TEMPLATE
 
 Options:
 
-  -machine-readable  Machine-readable output
+  -machine-readable             Machine-readable output
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

command/validate.go

@@ -78,6 +78,7 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 
 	diags = packerStarter.Initialize(packer.InitializeOptions{
 		SkipDatasourcesExecution: !cla.EvaluateDatasources,
+		UseSequential:            cla.UseSequential,
 	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
@@ -124,6 +125,7 @@ Options:
   -no-warn-undeclared-var       Disable warnings for user variable files containing undeclared variables.
   -evaluate-datasources         Evaluate data sources during validation (HCL2 only, may incur costs); Defaults to false. 
   -ignore-prerelease-plugins    Disable the loading of prerelease plugin binaries (x.y.z-dev).
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

hcl2template/parser.go

@@ -7,12 +7,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
 
 	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/ext/dynblock"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
+	"github.com/hashicorp/packer/internal/dag"
 	"github.com/hashicorp/packer/packer"
 	"github.com/zclconf/go-cty/cty"
 )
@@ -295,10 +297,224 @@ func filterVarsFromLogs(inputOrLocal Variables) {
 	}
 }
 
+func (cfg *PackerConfig) detectBuildPrereqDependencies() hcl.Diagnostics {
+	var diags hcl.Diagnostics
+
+	for _, ds := range cfg.Datasources {
+		dependencies := GetVarsByType(ds.block, "data")
+		dependencies = append(dependencies, GetVarsByType(ds.block, "local")...)
+
+		for _, dep := range dependencies {
+			// If something is locally aliased as `local` or `data`, we'll falsely
+			// report it as a local variable, which is not necessarily what we
+			// want to process here, so we continue.
+			//
+			// Note: this is kinda brittle, we should understand scopes to accurately
+			// mark something from an expression as a reference to a local variable.
+			// No real good solution for this now, besides maybe forbidding something
+			// to be locally aliased as `local`.
+			if len(dep) < 2 {
+				continue
+			}
+			rs, err := NewRefStringFromDep(dep)
+			if err != nil {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "failed to process datasource dependency",
+					Detail: fmt.Sprintf("An error occurred while processing a dependency for data source %s: %s",
+						ds.Name(), err),
+				})
+				continue
+			}
+
+			err = ds.RegisterDependency(rs)
+			if err != nil {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "failed to register datasource dependency",
+					Detail: fmt.Sprintf("An error occurred while registering %q as a dependency for data source %s: %s",
+						rs, ds.Name(), err),
+				})
+			}
+		}
+
+		cfg.Datasources[ds.Ref()] = ds
+	}
+
+	for _, loc := range cfg.LocalBlocks {
+		dependencies := FilterTraversalsByType(loc.Expr.Variables(), "data")
+		dependencies = append(dependencies, FilterTraversalsByType(loc.Expr.Variables(), "local")...)
+
+		for _, dep := range dependencies {
+			// If something is locally aliased as `local` or `data`, we'll falsely
+			// report it as a local variable, which is not necessarily what we
+			// want to process here, so we continue.
+			//
+			// Note: this is kinda brittle, we should understand scopes to accurately
+			// mark something from an expression as a reference to a local variable.
+			// No real good solution for this now, besides maybe forbidding something
+			// to be locally aliased as `local`.
+			if len(dep) < 2 {
+				continue
+			}
+			rs, err := NewRefStringFromDep(dep)
+			if err != nil {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "failed to process local dependency",
+					Detail: fmt.Sprintf("An error occurred while processing a dependency for local variable %s: %s",
+						loc.LocalName, err),
+				})
+				continue
+			}
+
+			err = loc.RegisterDependency(rs)
+			if err != nil {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "failed to register local dependency",
+					Detail: fmt.Sprintf("An error occurred while registering %q as a dependency for local variable %s: %s",
+						rs, loc.LocalName, err),
+				})
+			}
+		}
+	}
+
+	return diags
+}
+
+func (cfg *PackerConfig) buildPrereqsDAG() (*dag.AcyclicGraph, error) {
+	retGraph := dag.AcyclicGraph{}
+
+	verticesMap := map[string]dag.Vertex{}
+
+	// Do a first pass to create all the vertices
+	for ref := range cfg.Datasources {
+		// We keep a reference to the datasource separately from where it
+		// is used to avoid getting bit by the loop semantics.
+		//
+		// This `ds` local variable is the same object for every loop
+		// so if we directly use the address of this object, we'll end
+		// up referencing the last node of the loop for each vertex,
+		// leading to implicit cycles.
+		//
+		// However by capturing it locally in this loop, we have a
+		// reference to the actual datasource block, so it ends-up being
+		// the right instance for each vertex.
+		ds := cfg.Datasources[ref]
+		v := retGraph.Add(&ds)
+		verticesMap[fmt.Sprintf("data.%s", ds.Name())] = v
+	}
+	// Note: locals being references to the objects already, we can safely
+	// use the reference returned by the local loop.
+	for _, local := range cfg.LocalBlocks {
+		v := retGraph.Add(local)
+		verticesMap[fmt.Sprintf("local.%s", local.LocalName)] = v
+	}
+
+	// Connect the vertices together
+	//
+	// Vertices that don't have dependencies will be connected to the
+	// root vertex of the graph
+	for _, ds := range cfg.Datasources {
+		dsName := fmt.Sprintf("data.%s", ds.Name())
+
+		for _, dep := range ds.Dependencies {
+			retGraph.Connect(
+				dag.BasicEdge(verticesMap[dsName],
+					verticesMap[dep.String()]))
+		}
+	}
+	for _, loc := range cfg.LocalBlocks {
+		locName := fmt.Sprintf("local.%s", loc.LocalName)
+
+		for _, dep := range loc.dependencies {
+			retGraph.Connect(
+				dag.BasicEdge(verticesMap[locName],
+					verticesMap[dep.String()]))
+		}
+	}
+
+	if err := retGraph.Validate(); err != nil {
+		return nil, err
+	}
+
+	return &retGraph, nil
+}
+
+func (cfg *PackerConfig) evaluateBuildPrereqs(skipDatasources bool) hcl.Diagnostics {
+	diags := cfg.detectBuildPrereqDependencies()
+	if diags.HasErrors() {
+		return diags
+	}
+
+	graph, err := cfg.buildPrereqsDAG()
+	if err != nil {
+		return diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "failed to prepare execution graph",
+			Detail:   fmt.Sprintf("An error occurred while building the graph for datasources/locals: %s", err),
+		})
+	}
+
+	walkFunc := func(v dag.Vertex) hcl.Diagnostics {
+		var diags hcl.Diagnostics
+
+		switch bl := v.(type) {
+		case *DatasourceBlock:
+			diags = cfg.evaluateDatasource(*bl, skipDatasources)
+		case *LocalBlock:
+			var val *Variable
+			if cfg.LocalVariables == nil {
+				cfg.LocalVariables = make(Variables)
+			}
+			val, diags = cfg.evaluateLocalVariable(bl)
+			// Note: clumsy a bit, but we won't add the variable as `nil` here
+			// unless no errors have been reported during evaluation.
+			//
+			// This prevents Packer from panicking down the line, as initialisation
+			// doesn't stop if there are diags, so if `val` is nil, it crashes.
+			if !diags.HasErrors() {
+				cfg.LocalVariables[bl.LocalName] = val
+			}
+		default:
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "unsupported DAG node type",
+				Detail: fmt.Sprintf("A node of type %q was added to the DAG, but cannot be "+
+					"evaluated as it is unsupported. "+
+					"This is a Packer bug, please report it so we can investigate.",
+					reflect.TypeOf(v).String()),
+			})
+		}
+
+		if diags.HasErrors() {
+			return diags
+		}
+
+		return nil
+	}
+
+	for _, vtx := range graph.ReverseTopologicalOrder() {
+		vtxDiags := walkFunc(vtx)
+		if vtxDiags.HasErrors() {
+			diags = diags.Extend(vtxDiags)
+			return diags
+		}
+	}
+
+	return nil
+}
+
 func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics {
 	diags := cfg.InputVariables.ValidateValues()
-	diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...)
-	diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)
+
+	if opts.UseSequential {
+		diags = diags.Extend(cfg.evaluateDatasources(opts.SkipDatasourcesExecution))
+		diags = diags.Extend(cfg.evaluateLocalVariables(cfg.LocalBlocks))
+	} else {
+		diags = diags.Extend(cfg.evaluateBuildPrereqs(opts.SkipDatasourcesExecution))
+	}
 
 	filterVarsFromLogs(cfg.InputVariables)
 	filterVarsFromLogs(cfg.LocalVariables)