Introduce elasticsearch.ingest_pipeline.name as config option

In elastic#552 `elasticsearch` as config block is introduced. As discussed there, it also makes sense to use this for the Ingest Pipeline config to have all Elasticsearch related parts in one place.

For now no package uses this change and Kibana does not support it. As both ways are supported, this is not a breaking change yet. As a follow up, the package generation must be updated to adjust for this option and Kibana must be adapted to only support the new option.

CHANGELOG.md

@@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add list of downloads to /search endpoint. [#512](https://github.com/elastic/package-registry/pull/512)
 * Apply rule: first package found served. [#546](https://github.com/elastic/package-registry/pull/546)
 * Implement package watcher. [#553](https://github.com/elastic/package-registry/pull/553)
+* Introduce `elasticsearch.ingest_pipeline.name` as config option. [#](https://github.com/elastic/package-registry/pull/)
 
 ### Deprecated
 

util/dataset.go

@@ -32,13 +32,14 @@ var validTypes = map[string]string{
 }
 
 type DataSet struct {
-	ID             string   `config:"id" json:"id,omitempty" yaml:"id,omitempty"`
-	Title          string   `config:"title" json:"title" validate:"required"`
-	Release        string   `config:"release" json:"release"`
-	Type           string   `config:"type" json:"type" validate:"required"`
-	IngestPipeline string   `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty" yaml:"ingest_pipeline,omitempty"`
-	Streams        []Stream `config:"streams" json:"streams,omitempty" yaml:"streams,omitempty" `
-	Package        string   `json:"package,omitempty" yaml:"package,omitempty"`
+	ID             string         `config:"id" json:"id,omitempty" yaml:"id,omitempty"`
+	Title          string         `config:"title" json:"title" validate:"required"`
+	Release        string         `config:"release" json:"release"`
+	Type           string         `config:"type" json:"type" validate:"required"`
+	IngestPipeline string         `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty" yaml:"ingest_pipeline,omitempty"`
+	Streams        []Stream       `config:"streams" json:"streams,omitempty" yaml:"streams,omitempty" `
+	Package        string         `json:"package,omitempty" yaml:"package,omitempty"`
+	Elasticsearch  *Elasticsearch `config:"elasticsearch,omitempty" json:"elasticsearch,omitempty" yaml:"elasticsearch,omitempty"`
 
 	// Generated fields
 	Path string `json:"path,omitempty" yaml:"path,omitempty"`
@@ -84,6 +85,10 @@ type Os struct {
 	Windows interface{} `config:"windows" json:"windows,omitempty" yaml:"windows,omitempty"`
 }
 
+type Elasticsearch struct {
+	IngestPipelineName string `config:"ingest_pipeline.name,omitempty" json:"ingest_pipeline.name,omitempty" yaml:"ingest_pipeline.name,omitempty"`
+}
+
 type fieldEntry struct {
 	name  string
 	aType string
@@ -111,7 +116,7 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) {
 	}
 
 	// go-ucfg automatically calls the `Validate` method on the Dataset object here
-	err = manifest.Unpack(d)
+	err = manifest.Unpack(d, ucfg.PathSep("."))
 	if err != nil {
 		return nil, errors.Wrapf(err, "error building dataset (path: %s) in package: %s", datasetPath, p.Name)
 	}
@@ -154,7 +159,18 @@ func (d *DataSet) Validate() error {
 		return fmt.Errorf("type is not valid: %s", d.Type)
 	}
 
-	if d.IngestPipeline == "" {
+	if d.Elasticsearch != nil && d.Elasticsearch.IngestPipelineName == "" {
+		// Check that no ingest pipeline exists in the directory except default
+		for _, path := range paths {
+			if filepath.Base(path) == "default.json" || filepath.Base(path) == "default.yml" {
+				d.Elasticsearch.IngestPipelineName = "default"
+				// TODO: remove because of legacy
+				d.IngestPipeline = "default"
+				break
+			}
+		}
+		// TODO: Remove, only here for legacy
+	} else if d.IngestPipeline == "" {
 		// Check that no ingest pipeline exists in the directory except default
 		for _, path := range paths {
 			if filepath.Base(path) == "default.json" || filepath.Base(path) == "default.yml" {