Move light modules to OSS (#14369)

CHANGELOG-developer.next.asciidoc

@@ -26,6 +26,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509]
 - The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610]
 - Altered all remaining uses of mapval to use the renamed and enhanced version: https://github.com/elastic/go-lookslike[go-lookslike] instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165]
+- Move light modules to OSS. {pull}14369[14369]
 
 ==== Bugfixes
 

metricbeat/beater/metricbeat.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/common/reload"
 	"github.com/elastic/beats/libbeat/management"
+	"github.com/elastic/beats/libbeat/paths"
 
 	"github.com/joeshaw/multierror"
 	"github.com/pkg/errors"
@@ -69,6 +70,14 @@ func WithModuleOptions(options ...module.Option) Option {
 	}
 }
 
+// WithLightModules enables light modules support
+func WithLightModules() Option {
+	return func(*Metricbeat) {
+		path := paths.Resolve(paths.Home, "module")
+		mb.Registry.SetSecondarySource(mb.NewLightModulesSource(path))
+	}
+}
+
 // Creator returns a beat.Creator for instantiating a new instance of the
 // Metricbeat framework with the given options.
 func Creator(options ...Option) beat.Creator {
@@ -90,13 +99,38 @@ func Creator(options ...Option) beat.Creator {
 //     )
 func DefaultCreator() beat.Creator {
 	return Creator(
+		WithLightModules(),
 		WithModuleOptions(
 			module.WithMetricSetInfo(),
 			module.WithServiceName(),
 		),
 	)
 }
 
+// DefaultTestModulesCreator returns a customized instance of Metricbeat
+// where startup delay has been disabled to workaround the fact that
+// Modules() will return the static modules (not the dynamic ones)
+// with a start delay.
+//
+// This is equivalent to calling
+//
+//  beater.Creator(
+//		beater.WithLightModules(),
+//		beater.WithModuleOptions(
+//			module.WithMetricSetInfo(),
+//			module.WithMaxStartDelay(0),
+//		),
+//	)
+func DefaultTestModulesCreator() beat.Creator {
+	return Creator(
+		WithLightModules(),
+		WithModuleOptions(
+			module.WithMetricSetInfo(),
+			module.WithMaxStartDelay(0),
+		),
+	)
+}
+
 // newMetricbeat creates and returns a new Metricbeat instance.
 func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbeat, error) {
 	config := defaultConfig

metricbeat/cmd/root.go

@@ -22,12 +22,10 @@ import (
 
 	"github.com/spf13/pflag"
 
-	cmd "github.com/elastic/beats/libbeat/cmd"
+	"github.com/elastic/beats/libbeat/cmd"
 	"github.com/elastic/beats/libbeat/cmd/instance"
 	"github.com/elastic/beats/metricbeat/beater"
 	"github.com/elastic/beats/metricbeat/cmd/test"
-	"github.com/elastic/beats/metricbeat/mb/module"
-
 	// import modules
 	_ "github.com/elastic/beats/metricbeat/include"
 	_ "github.com/elastic/beats/metricbeat/include/fields"
@@ -39,22 +37,10 @@ var Name = "metricbeat"
 // RootCmd to handle beats cli
 var RootCmd *cmd.BeatsRootCmd
 
-var (
-	// Use a customized instance of Metricbeat where startup delay has
-	// been disabled to workaround the fact that Modules() will return
-	// the static modules (not the dynamic ones) with a start delay.
-	testModulesCreator = beater.Creator(
-		beater.WithModuleOptions(
-			module.WithMetricSetInfo(),
-			module.WithMaxStartDelay(0),
-		),
-	)
-)
-
 func init() {
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs"))
 	RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{RunFlags: runFlags, Name: Name})
 	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager))
-	RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator))
+	RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator()))
 }

x-pack/metricbeat/mb/lightmodules.go → metricbeat/mb/lightmodules.go

@@ -1,6 +1,19 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 
 package mb
 
@@ -15,7 +28,6 @@ import (
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
-	"github.com/elastic/beats/metricbeat/mb"
 )
 
 const (
@@ -105,15 +117,15 @@ func (s *LightModulesSource) HasMetricSet(moduleName, metricSetName string) bool
 }
 
 // MetricSetRegistration obtains a registration for a light metric set
-func (s *LightModulesSource) MetricSetRegistration(register *mb.Register, moduleName, metricSetName string) (mb.MetricSetRegistration, error) {
+func (s *LightModulesSource) MetricSetRegistration(register *Register, moduleName, metricSetName string) (MetricSetRegistration, error) {
 	lightModule, err := s.loadModule(moduleName)
 	if err != nil {
-		return mb.MetricSetRegistration{}, errors.Wrapf(err, "failed to load module '%s'", moduleName)
+		return MetricSetRegistration{}, errors.Wrapf(err, "failed to load module '%s'", moduleName)
 	}
 
 	ms, found := lightModule.MetricSets[metricSetName]
 	if !found {
-		return mb.MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' not found", moduleName, metricSetName)
+		return MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' not found", moduleName, metricSetName)
 	}
 
 	return ms.Registration(register)
@@ -141,7 +153,7 @@ type lightModuleConfig struct {
 // LightModule contains the definition of a light module
 type LightModule struct {
 	Name       string
-	MetricSets map[string]mb.LightMetricSet
+	MetricSets map[string]LightMetricSet
 }
 
 func (s *LightModulesSource) loadModule(moduleName string) (*LightModule, error) {
@@ -186,8 +198,8 @@ func (s *LightModulesSource) loadModuleConfig(modulePath string) (*lightModuleCo
 	return &moduleConfig, nil
 }
 
-func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, metricSetNames []string) (map[string]mb.LightMetricSet, error) {
-	metricSets := make(map[string]mb.LightMetricSet)
+func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, metricSetNames []string) (map[string]LightMetricSet, error) {
+	metricSets := make(map[string]LightMetricSet)
 	for _, metricSet := range metricSetNames {
 		manifestPath := filepath.Join(moduleDirPath, metricSet, manifestYML)
 
@@ -203,7 +215,7 @@ func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, me
 	return metricSets, nil
 }
 
-func (s *LightModulesSource) loadMetricSetConfig(manifestPath string) (ms mb.LightMetricSet, err error) {
+func (s *LightModulesSource) loadMetricSetConfig(manifestPath string) (ms LightMetricSet, err error) {
 	config, err := common.LoadFile(manifestPath)
 	if err != nil {
 		return ms, errors.Wrapf(err, "failed to load metricset manifest from '%s'", manifestPath)

x-pack/metricbeat/mb/lightmodules_test.go → metricbeat/mb/lightmodules_test.go

@@ -1,6 +1,19 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 
 // +build !integration
 
@@ -15,7 +28,6 @@ import (
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
-	"github.com/elastic/beats/metricbeat/mb"
 )
 
 // TestLightModulesAsModuleSource checks that registry correctly lists
@@ -27,7 +39,7 @@ func TestLightModulesAsModuleSource(t *testing.T) {
 		name       string
 		module     string
 		isDefault  bool
-		hostParser mb.HostParser
+		hostParser HostParser
 	}
 
 	cases := map[string]struct {
@@ -77,19 +89,19 @@ func TestLightModulesAsModuleSource(t *testing.T) {
 		},
 	}
 
-	fakeMetricSetFactory := func(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	fakeMetricSetFactory := func(base BaseMetricSet) (MetricSet, error) {
 		return &base, nil
 	}
 
-	newRegistry := func(metricSets []testMetricSet) *mb.Register {
-		r := mb.NewRegister()
+	newRegistry := func(metricSets []testMetricSet) *Register {
+		r := NewRegister()
 		for _, m := range metricSets {
-			opts := []mb.MetricSetOption{}
+			opts := []MetricSetOption{}
 			if m.isDefault {
-				opts = append(opts, mb.DefaultMetricSet())
+				opts = append(opts, DefaultMetricSet())
 			}
 			if m.hostParser != nil {
-				opts = append(opts, mb.WithHostParser(m.hostParser))
+				opts = append(opts, WithHostParser(m.hostParser))
 			}
 			r.MustAddMetricSet(m.module, m.name, fakeMetricSetFactory, opts...)
 		}
@@ -174,7 +186,7 @@ func TestNewModuleFromConfig(t *testing.T) {
 		config         common.MapStr
 		err            bool
 		expectedOption string
-		expectedQuery  mb.QueryParams
+		expectedQuery  QueryParams
 		expectedPeriod time.Duration
 	}{
 		"normal module": {
@@ -196,7 +208,7 @@ func TestNewModuleFromConfig(t *testing.T) {
 		"light module with query": {
 			config:         common.MapStr{"module": "service", "query": common.MapStr{"param": "foo"}},
 			expectedOption: "test",
-			expectedQuery:  mb.QueryParams{"param": "foo"},
+			expectedQuery:  QueryParams{"param": "foo"},
 		},
 		"light module with custom period": {
 			config:         common.MapStr{"module": "service", "period": "42s"},
@@ -217,7 +229,7 @@ func TestNewModuleFromConfig(t *testing.T) {
 		},
 	}
 
-	r := mb.NewRegister()
+	r := NewRegister()
 	r.MustAddMetricSet("foo", "bar", newMetricSetWithOption)
 	r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules"))
 
@@ -226,7 +238,7 @@ func TestNewModuleFromConfig(t *testing.T) {
 			config, err := common.NewConfigFrom(c.config)
 			require.NoError(t, err)
 
-			module, metricSets, err := mb.NewModule(config, r)
+			module, metricSets, err := NewModule(config, r)
 			if c.err {
 				assert.Error(t, err)
 				return
@@ -248,7 +260,7 @@ func TestNewModuleFromConfig(t *testing.T) {
 					assert.Equal(t, c.expectedQuery, ms.Module().Config().Query)
 					expectedPeriod := c.expectedPeriod
 					if expectedPeriod == 0 {
-						expectedPeriod = mb.DefaultModuleConfig().Period
+						expectedPeriod = DefaultModuleConfig().Period
 					}
 					assert.Equal(t, expectedPeriod, ms.Module().Config().Period)
 				})
@@ -260,31 +272,31 @@ func TestNewModuleFromConfig(t *testing.T) {
 func TestNewModulesCallModuleFactory(t *testing.T) {
 	logp.TestingSetup()
 
-	r := mb.NewRegister()
+	r := NewRegister()
 	r.MustAddMetricSet("foo", "bar", newMetricSetWithOption)
 	r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules"))
 
 	called := false
-	r.AddModule("foo", func(base mb.BaseModule) (mb.Module, error) {
+	r.AddModule("foo", func(base BaseModule) (Module, error) {
 		called = true
-		return mb.DefaultModuleFactory(base)
+		return DefaultModuleFactory(base)
 	})
 
 	config, err := common.NewConfigFrom(common.MapStr{"module": "service"})
 	require.NoError(t, err)
 
-	_, _, err = mb.NewModule(config, r)
+	_, _, err = NewModule(config, r)
 	assert.NoError(t, err)
 
 	assert.True(t, called, "module factory must be called if registered")
 }
 
 type metricSetWithOption struct {
-	mb.BaseMetricSet
+	BaseMetricSet
 	Option string
 }
 
-func newMetricSetWithOption(base mb.BaseMetricSet) (mb.MetricSet, error) {
+func newMetricSetWithOption(base BaseMetricSet) (MetricSet, error) {
 	config := struct {
 		Option string `config:"option"`
 	}{
@@ -301,4 +313,4 @@ func newMetricSetWithOption(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	}, nil
 }
 
-func (*metricSetWithOption) Fetch(mb.ReporterV2) error { return nil }
+func (*metricSetWithOption) Fetch(ReporterV2) error { return nil }