[Elastic Agent] Add support for variable replacement from providers (e…

…lastic#20839)

* Add ability to replace variables in the parsed AST tree.

* More vars replace improvements.

* Perform the variable replacement of the elastic-agent configuration.

* Clean-up testing and processors.

* Add changelog.

* Fix import sorting.

* Add more validation to variable substitution.

* Add log message about dynamic inputs being experimental.

* Update to new variable format. Handle replace of complete objects.

* Fix config importing to not replace vars in inputs.

* Fixes for vet.

* Fix fleet config change action to use new LoadConfig.

* Fixes from code review.

* Ensure processors are prepended to inputs.

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -17,3 +17,4 @@
 - Add restart CLI cmd {pull}20359[20359]
 - Add new `synthetics/*` inputs to run Heartbeat {pull}20387[20387]
 - Users of the Docker image can now pass `FLEET_ENROLL_INSECURE=1` to include the `--insecure` flag with the `elastic-agent enroll` command {issue}20312[20312] {pull}20713[20713]
+- Add support for dynamic inputs with providers and `{{variable|"default"}}` substitution. {pull}20839[20839]

x-pack/elastic-agent/pkg/agent/application/application.go

@@ -26,7 +26,7 @@ func New(log *logger.Logger, pathConfigFile string) (Application, error) {
 	// Load configuration from disk to understand in which mode of operation
 	// we must start the elastic-agent, the mode of operation cannot be changed without restarting the
 	// elastic-agent.
-	rawConfig, err := config.LoadYAML(pathConfigFile)
+	rawConfig, err := LoadConfigFromFile(pathConfigFile)
 	if err != nil {
 		return nil, err
 	}

x-pack/elastic-agent/pkg/agent/application/config.go

@@ -5,8 +5,15 @@
 package application
 
 import (
+	"io/ioutil"
+
+	"github.com/elastic/go-ucfg"
+
+	"gopkg.in/yaml.v2"
+
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 )
 
@@ -26,3 +33,50 @@ func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*conf
 	}
 	return cfg, nil
 }
+
+// LoadConfigFromFile loads the Agent configuration from a file.
+//
+// This must be used to load the Agent configuration, so that variables defined in the inputs are not
+// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
+func LoadConfigFromFile(path string) (*config.Config, error) {
+	in, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var m map[string]interface{}
+	if err := yaml.Unmarshal(in, &m); err != nil {
+		return nil, err
+	}
+	return LoadConfig(m)
+}
+
+// LoadConfig loads the Agent configuration from a map.
+//
+// This must be used to load the Agent configuration, so that variables defined in the inputs are not
+// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
+func LoadConfig(m map[string]interface{}) (*config.Config, error) {
+	inputs, ok := m["inputs"]
+	if ok {
+		// remove the inputs
+		delete(m, "inputs")
+	}
+	cfg, err := config.NewConfigFrom(m)
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		inputsOnly := map[string]interface{}{
+			"inputs": inputs,
+		}
+		// convert to config without variable substitution
+		inputsCfg, err := config.NewConfigFrom(inputsOnly, ucfg.PathSep("."), ucfg.ResolveNOOP)
+		if err != nil {
+			return nil, err
+		}
+		err = cfg.Merge(inputsCfg, ucfg.PathSep("."), ucfg.ResolveNOOP)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return cfg, err
+}

x-pack/elastic-agent/pkg/agent/application/config_test.go

@@ -5,16 +5,59 @@
 package application
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 )
 
+func TestLoadConfig(t *testing.T) {
+	contents := map[string]interface{}{
+		"outputs": map[string]interface{}{
+			"default": map[string]interface{}{
+				"type":     "elasticsearch",
+				"hosts":    []interface{}{"127.0.0.1:9200"},
+				"username": "elastic",
+				"password": "changeme",
+			},
+		},
+		"inputs": []interface{}{
+			map[string]interface{}{
+				"type": "logfile",
+				"streams": []interface{}{
+					map[string]interface{}{
+						"paths": []interface{}{"/var/log/${host.name}"},
+					},
+				},
+			},
+		},
+	}
+
+	tmp, err := ioutil.TempDir("", "config")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmp)
+
+	cfgPath := filepath.Join(tmp, "config.yml")
+	dumpToYAML(t, cfgPath, contents)
+
+	cfg, err := LoadConfigFromFile(cfgPath)
+	require.NoError(t, err)
+
+	cfgData, err := cfg.ToMapStr()
+	require.NoError(t, err)
+
+	assert.Equal(t, contents, cfgData)
+}
+
 func TestConfig(t *testing.T) {
 	testMgmtMode(t)
 	testLocalConfig(t)
@@ -74,3 +117,9 @@ func mustWithConfigMode(standalone bool) *config.Config {
 		},
 	)
 }
+
+func dumpToYAML(t *testing.T, out string, in interface{}) {
+	b, err := yaml.Marshal(in)
+	require.NoError(t, err)
+	ioutil.WriteFile(out, b, 0600)
+}