Skip to content

Commit

Permalink
Fix add_labels flattening of arrays values
Browse files Browse the repository at this point in the history
The processor was not working as described in the docs. For example:

```yaml
processors:
  - add_labels:
      labels:
        number: 1
        with.dots: test
        nested:
          with.dots: nested
        array:
          - do
          - re
          - with.field: mi
```

```json
{
  "labels": {
    "array": [
      "do",
      "re",
      {
        "with": {
          "field": "mi"
        }
      }
    ],
    "nested.with.dots": "nested",
    "number": 1,
    "with.dots": "test"
  }
```

```json
{
  "labels": {
    "number": 1,
    "with.dots": "test",
    "nested.with.dots": "nested",
    "array.0": "do",
    "array.1": "re",
    "array.2.with.field": "mi"
  }
}
```
  • Loading branch information
andrewkroh committed Dec 1, 2021
1 parent 51463bf commit 4a62f82
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 6 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix the wrong beat name on monitoring and state endpoint {issue}27755[27755]
- Skip configuration checks in autodiscover for configurations that are already running {pull}29048[29048]
- Fix `decode_json_processor` to always respect `add_error_key` {pull}29107[29107]
- Fix `add_labels` flattening of array values. {pull}29211[29211]

*Auditbeat*

Expand Down Expand Up @@ -188,7 +189,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Revert usageDetails api version to 2019-01-01. {pull}28995[28995]
- Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
- Fix `threatintel.misp` filters configuration. {issue}27970[27970]
- Fix opening files on Windows in filestream so open files can be deleted. {issue}29113[29113] {pull}29180[29180]
- Fix opening files on Windows in filestream so open files can be deleted. {issue}29113[29113] {pull}29180[29180]

*Heartbeat*

Expand Down
42 changes: 37 additions & 5 deletions libbeat/processors/actions/add_labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ package actions
import (
"fmt"

"github.com/elastic/go-ucfg"

"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/libbeat/processors"
"github.com/elastic/beats/v7/libbeat/processors/checks"
Expand All @@ -41,10 +43,15 @@ func createAddLabels(c *common.Config) (processors.Processor, error) {
}{}
err := c.Unpack(&config)
if err != nil {
return nil, fmt.Errorf("fail to unpack the add_fields configuration: %s", err)
return nil, fmt.Errorf("fail to unpack the add_fields configuration: %w", err)
}

flatLabels, err := flattenLabels(config.Labels)
if err != nil {
return nil, fmt.Errorf("failed to flatten labels: %w", err)
}

return makeFieldsProcessor(LabelsKey, config.Labels.Flatten(), true), nil
return makeFieldsProcessor(LabelsKey, flatLabels, true), nil
}

// NewAddLabels creates a new processor adding the given object to events. Set
Expand All @@ -53,8 +60,33 @@ func createAddLabels(c *common.Config) (processors.Processor, error) {
// If labels contains nested objects, NewAddLabels will flatten keys into labels by
// by joining names with a dot ('.') .
// The labels will be inserted into the 'labels' field.
func NewAddLabels(labels common.MapStr, shared bool) processors.Processor {
func NewAddLabels(labels common.MapStr, shared bool) (processors.Processor, error) {
flatLabels, err := flattenLabels(labels)
if err != nil {
return nil, fmt.Errorf("failed to flatten labels: %w", err)
}

return NewAddFields(common.MapStr{
LabelsKey: labels.Flatten(),
}, shared, true)
LabelsKey: flatLabels,
}, shared, true), nil
}

func flattenLabels(labels common.MapStr) (common.MapStr, error) {
labelConfig, err := common.NewConfigFrom(labels)
if err != nil {
return nil, err
}

flatKeys := (*ucfg.Config)(labelConfig).FlattenedKeys()

flatMap := make(common.MapStr, len(flatKeys))
for _, k := range flatKeys {
v, err := labelConfig.String(k, -1)
if err != nil {
return nil, err
}
flatMap[k] = v
}

return flatMap, nil
}
11 changes: 11 additions & 0 deletions libbeat/processors/actions/add_labels_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,16 @@ func TestAddLabels(t *testing.T) {
`{add_labels.labels: {l2: b, lc: b}}`,
),
},
"add array": {
event: common.MapStr{},
want: common.MapStr{
"labels": common.MapStr{
"array.0": "foo",
"array.1": "bar",
"array.2.hello": "world",
},
},
cfg: single(`{add_labels: {labels: {array: ["foo", "bar", {"hello": "world"}]}}}`),
},
})
}

0 comments on commit 4a62f82

Please sign in to comment.