Add --force-enable-module-filesets option to filebeat setup (elastic#…

…36286)

* Add --force-enable-module-filesets option to filebeat setup

Closes elastic#30916

CHANGELOG.next.asciidoc

@@ -236,6 +236,8 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Added support for Okta OAuth2 provider in the httpjson input. {pull}36273[36273]
 - Add support of the interval parameter in Salesforce setupaudittrail-rest fileset. {issue}35917[35917] {pull}35938[35938]
 - Add device handling to Okta input package for entity analytics. {pull}36049[36049]
+- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999]
+- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}36286[36286]
 
 *Auditbeat*
 

NOTICE.txt

@@ -21183,6 +21183,37 @@ The above copyright notice and this permission notice shall be included in all c
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/copy
+Version: v1.12.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/[email protected]/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2018 otiai10
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/pierrec/lz4/v4
 Version: v4.1.15
@@ -46273,6 +46304,23 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-sp
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/mint
+Version: v1.5.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/[email protected]/LICENSE:
+
+Copyright 2017 otiai10 (Hiromu OCHIAI)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/oxtoacart/bpool
 Version: v0.0.0-20150712133111-4e1c5567d7c2

filebeat/autodiscover/builder/hints/logs.go

@@ -69,7 +69,7 @@ func NewLogHints(cfg *conf.C) (autodiscover.Builder, error) {
 		return nil, fmt.Errorf("unable to unpack hints config due to error: %w", err)
 	}
 
-	moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, false)
+	moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, fileset.FilesetOverrides{})
 	if err != nil {
 		return nil, err
 	}
@@ -105,7 +105,7 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf
 		return template.ApplyConfigTemplate(event, configs)
 	}
 
-	var configs []*conf.C
+	var configs []*conf.C //nolint:prealloc //breaks tests
 	inputs := l.getInputs(hints)
 	for _, h := range inputs {
 		// Clone original config, enable it if disabled
@@ -262,7 +262,7 @@ func (l *logHints) getFilesets(hints mapstr.M, module string) map[string]*filese
 
 func (l *logHints) getInputs(hints mapstr.M) []mapstr.M {
 	modules := utils.GetHintsAsList(hints, l.config.Key)
-	var output []mapstr.M
+	var output []mapstr.M //nolint:prealloc //breaks tests
 
 	for _, mod := range modules {
 		output = append(output, mapstr.M{

filebeat/beater/filebeat.go

@@ -108,7 +108,13 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
 		return nil, err
 	}
 
-	moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, false)
+	enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1)
+	forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1)
+	filesetOverrides := fileset.FilesetOverrides{
+		EnableAllFilesets:         enableAllFilesets,
+		ForceEnableModuleFilesets: forceEnableModuleFilesets,
+	}
+	moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, filesetOverrides)
 	if err != nil {
 		return nil, err
 	}
@@ -199,7 +205,13 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error {
 		// have to be loaded using cfg.Reloader. Otherwise those configurations are skipped.
 		pipelineLoaderFactory := newPipelineLoaderFactory(b.Config.Output.Config())
 		enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1)
-		modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, enableAllFilesets)
+		forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1)
+		filesetOverrides := fileset.FilesetOverrides{
+			EnableAllFilesets:         enableAllFilesets,
+			ForceEnableModuleFilesets: forceEnableModuleFilesets,
+		}
+
+		modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, filesetOverrides)
 		if fb.config.ConfigModules.Enabled() {
 			if enableAllFilesets {
 				// All module configs need to be loaded to enable all the filesets

filebeat/docs/howto/load-ingest-pipelines.asciidoc

@@ -5,7 +5,7 @@ The ingest pipelines used to parse log lines are set up automatically the first
 time you run {beatname_uc}, assuming the {es} output is enabled. If you're sending
 events to {ls} you need to load the ingest pipelines manually. To do this, run the
 `setup` command with the `--pipelines` option specified.  You also need to enable
-the modules and filesets, this can be accomplished one of two ways.
+the modules and filesets, this can be accomplished several ways.
 
 First you can use the `--modules` option to enable the module, and the
 `-M` option to enable the fileset.  For example, the following command
@@ -39,7 +39,40 @@ loads the access pipeline from the nginx module.
 PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules nginx -M "nginx.access.enabled=true"
 ----
 
-The second option is to use the `--enable-all-filesets` option to
+The second option is to use the `--modules` option to enable the
+module, and the `--force-enable-module-filesets` option to enable all
+the filesets in the module.  For example, the following command loads
+the access pipeline from the nginx module.
+
+*deb and rpm:*
+
+["source","sh",subs="attributes"]
+----
+{beatname_lc} setup --pipelines --modules nginx --force-enable-module-filesets
+----
+
+*mac:*
+
+["source","sh",subs="attributes"]
+----
+./{beatname_lc} setup --pipelines --modules nginx --force-enable-module-filesets
+----
+
+*linux:*
+
+["source","sh",subs="attributes"]
+----
+./{beatname_lc} setup --pipelines --modules nginx --force-enable-module-filesets
+----
+
+*win:*
+
+["source","sh",subs="attributes"]
+----
+PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules nginx --force-enable-module-filesets
+----
+
+The third option is to use the `--enable-all-filesets` option to
 enable all the modules and all the filesets so all of the ingest
 pipelines are loaded.  For example, the following command loads all
 the ingest pipelines.

filebeat/fileset/factory.go

@@ -129,7 +129,7 @@ func (f *Factory) CheckConfig(c *conf.C) error {
 // createRegistry starts a registry for a set of filesets, it returns the registry and
 // its input configurations
 func (f *Factory) createRegistry(c *conf.C) (*ModuleRegistry, []*conf.C, error) {
-	m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, false)
+	m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, FilesetOverrides{})
 	if err != nil {
 		return nil, nil, err
 	}

filebeat/fileset/modules.go

@@ -46,12 +46,17 @@ type Module struct {
 	config   ModuleConfig
 }
 
+type FilesetOverrides struct {
+	EnableAllFilesets         bool
+	ForceEnableModuleFilesets bool
+}
+
 // newModuleRegistry reads and loads the configured module into the registry.
 func newModuleRegistry(modulesPath string,
 	moduleConfigs []*ModuleConfig,
 	overrides *ModuleOverrides,
 	beatInfo beat.Info,
-	enableAllFilesets bool,
+	filesetOverrides FilesetOverrides,
 ) (*ModuleRegistry, error) {
 	reg := ModuleRegistry{
 		registry: []Module{},
@@ -61,7 +66,7 @@ func newModuleRegistry(modulesPath string,
 	for _, mcfg := range moduleConfigs {
 		// an empty ModuleConfig can reach this so we only force enable a
 		// config if the Module name is set and Enabled pointer is valid.
-		if enableAllFilesets && mcfg.Module != "" && mcfg.Enabled != nil {
+		if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && mcfg.Module != "" && mcfg.Enabled != nil {
 			*mcfg.Enabled = true
 		}
 		if mcfg.Module == "" || (mcfg.Enabled != nil && !(*mcfg.Enabled)) {
@@ -80,16 +85,26 @@ func newModuleRegistry(modulesPath string,
 			config:   *mcfg,
 			filesets: []Fileset{},
 		}
-		for filesetName, fcfg := range mcfg.Filesets {
+		if filesetOverrides.ForceEnableModuleFilesets {
+			if mcfg.Filesets == nil {
+				mcfg.Filesets = make(map[string]*FilesetConfig)
+			}
+			for _, fName := range moduleFilesets {
+				if _, ok := mcfg.Filesets[fName]; !ok {
+					mcfg.Filesets[fName] = &FilesetConfig{Enabled: func() *bool { b := true; return &b }()}
+				}
+			}
+		}
 
+		for filesetName, fcfg := range mcfg.Filesets {
 			fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides)
 			if err != nil {
 				return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %w", mcfg.Module, filesetName, err)
 			}
 
 			// ModuleConfig can have empty Filesets so we only force
 			// enable if the Enabled pointer is valid
-			if enableAllFilesets && fcfg.Enabled != nil {
+			if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && fcfg.Enabled != nil {
 				*fcfg.Enabled = true
 			}
 			if fcfg.Enabled != nil && !(*fcfg.Enabled) {
@@ -128,14 +143,14 @@ func newModuleRegistry(modulesPath string,
 }
 
 // NewModuleRegistry reads and loads the configured module into the registry.
-func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, enableAllFilesets bool) (*ModuleRegistry, error) {
+func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, filesetOverrides FilesetOverrides) (*ModuleRegistry, error) {
 	modulesPath := paths.Resolve(paths.Home, "module")
 
 	stat, err := os.Stat(modulesPath)
 	if err != nil || !stat.IsDir() {
 		log := logp.NewLogger(logName)
 		log.Errorf("Not loading modules. Module directory not found: %s", modulesPath)
-		return &ModuleRegistry{log: log}, nil // empty registry, no error
+		return &ModuleRegistry{log: log}, nil //nolint:nilerr // empty registry, no error
 	}
 
 	var modulesCLIList []string
@@ -146,7 +161,7 @@ func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, e
 			return nil, err
 		}
 	}
-	var mcfgs []*ModuleConfig
+	var mcfgs []*ModuleConfig //nolint:prealloc  //breaks tests
 	for _, cfg := range moduleConfigs {
 		cfg, err = mergePathDefaults(cfg)
 		if err != nil {
@@ -166,7 +181,7 @@ func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, e
 	}
 
 	enableFilesetsFromOverrides(mcfgs, modulesOverrides)
-	return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, enableAllFilesets)
+	return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, filesetOverrides)
 }
 
 // enableFilesetsFromOverrides enables in mcfgs the filesets mentioned in overrides,
@@ -433,7 +448,7 @@ func (reg *ModuleRegistry) Empty() bool {
 
 // ModuleNames returns the names of modules in the ModuleRegistry.
 func (reg *ModuleRegistry) ModuleNames() []string {
-	var modules []string
+	var modules []string //nolint:prealloc  //breaks tests
 	for _, m := range reg.registry {
 		modules = append(modules, m.config.Module)
 	}

filebeat/fileset/modules_integration_test.go

@@ -46,7 +46,7 @@ func TestLoadPipeline(t *testing.T) {
 		t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion())
 	}
 
-	client.Request("DELETE", "/_ingest/pipeline/my-pipeline-id", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/my-pipeline-id", "", nil, nil)
 
 	content := map[string]interface{}{
 		"description": "describe pipeline",
@@ -98,8 +98,8 @@ func TestSetupNginx(t *testing.T) {
 		t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion())
 	}
 
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-default", "", nil, nil)
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-default", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)
 
 	modulesPath, err := filepath.Abs("../module")
 	require.NoError(t, err)
@@ -114,7 +114,7 @@ func TestSetupNginx(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), FilesetOverrides{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -176,9 +176,9 @@ func TestLoadMultiplePipelines(t *testing.T) {
 		t.Skip("Skip tests because ingest is missing the pipeline processor: ", client.GetVersion())
 	}
 
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-pipeline", "", nil, nil)
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-json_logs", "", nil, nil)
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-plain_logs", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-pipeline", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-json_logs", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-plain_logs", "", nil, nil)
 
 	modulesPath, err := filepath.Abs("../_meta/test/module")
 	require.NoError(t, err)
@@ -193,7 +193,7 @@ func TestLoadMultiplePipelines(t *testing.T) {
 		{"foo", &enabled, filesetConfigs},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -221,9 +221,9 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) {
 		t.Skip("Skip tests because ingest is missing the pipeline processor: ", client.GetVersion())
 	}
 
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-pipeline", "", nil, nil)
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-json_logs", "", nil, nil)
-	client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-plain_logs_bad", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-pipeline", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-json_logs", "", nil, nil)
+	_, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-plain_logs_bad", "", nil, nil)
 
 	modulesPath, err := filepath.Abs("../_meta/test/module")
 	require.NoError(t, err)
@@ -238,7 +238,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) {
 		{"foo", &enabled, filesetConfigs},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{})
 	if err != nil {
 		t.Fatal(err)
 	}

filebeat/fileset/modules_test.go

@@ -81,7 +81,7 @@ func TestNewModuleRegistry(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 
@@ -148,7 +148,7 @@ func TestNewModuleRegistryConfig(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 
@@ -174,7 +174,7 @@ func TestMovedModule(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 }
@@ -445,7 +445,7 @@ func TestMissingModuleFolder(t *testing.T) {
 		load(t, map[string]interface{}{"module": "nginx"}),
 	}
 
-	reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, false)
+	reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)