From e114aeb7aaa3b93e875e3daad9f8d8643dbf20fc Mon Sep 17 00:00:00 2001 From: Chao Date: Sun, 5 Apr 2020 01:03:54 +0800 Subject: [PATCH 1/7] add urldecode processor --- CHANGELOG.next.asciidoc | 3 +- filebeat/tests/system/test_processors.py | 26 +++ libbeat/docs/processors-list.asciidoc | 6 + .../actions/docs/urldecode.asciidoc | 38 ++++ libbeat/processors/actions/urldecode.go | 127 +++++++++++ libbeat/processors/actions/urldecode_test.go | 214 ++++++++++++++++++ 6 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 libbeat/processors/actions/docs/urldecode.asciidoc create mode 100644 libbeat/processors/actions/urldecode.go create mode 100644 libbeat/processors/actions/urldecode_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 91798059ffb..758ec3d8bd3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -137,7 +137,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Use max in k8s overview dashboard aggregations. {pull}17015[17015] - Fix Disk Used and Disk Usage visualizations in the Metricbeat System dashboards. {issue}12435[12435] {pull}17272[17272] - Fix missing Accept header for Prometheus and OpenMetrics module. {issue}16870[16870] {pull}17291[17291] -- Further revise check for bad data in docker/memory. {pull}17400[17400] +- Further revise check for bad data in docker/memory. {pull}17400[17400] - Fix issue in Jolokia module when mbean contains multiple quoted properties. {issue}17375[17375] {pull}17374[17374] - Combine cloudwatch aggregated metrics into single event. {pull}17345[17345] - check if cpuOptions field is nil in DescribeInstances output in ec2 metricset. {pull}17418[17418] @@ -175,6 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update supported versions of `redis` output. {pull}17198[17198] - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268] - Add optional regex based cid extractor to `add_kubernetes_metadata` processor. {pull}17360[17360] +- Add `urldecode` processor to for decoding URL encoded fields. *Auditbeat* diff --git a/filebeat/tests/system/test_processors.py b/filebeat/tests/system/test_processors.py index dae4698a0c9..1dc0b975138 100644 --- a/filebeat/tests/system/test_processors.py +++ b/filebeat/tests/system/test_processors.py @@ -302,6 +302,32 @@ def test_decode_csv_fields_all_options(self): ["42", "hello world", "string\twith tabs and \"broken\" quotes"], ]) + def test_urldecode_defaults(self): + """ + Check URL-decoding using defaults + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/test.log", + processors=[{ + "urldecode": { + "fields": [{ + "from": "message", + "to": "decoded" + }] + }, + }] + ) + + self._init_and_read_test_input([ + "correct data\n", + "correct%20data\n", + ]) + + self._assert_expected_lines([ + "correct data", + "correct data", + ], field="decoded") + def test_javascript_processor_add_host_metadata(self): """ Check JS processor with add_host_metadata diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc index bebffb9d30c..5f78fc0dbea 100644 --- a/libbeat/docs/processors-list.asciidoc +++ b/libbeat/docs/processors-list.asciidoc @@ -101,6 +101,9 @@ endif::[] ifndef::no_translate_sid_processor[] * <> endif::[] +ifndef::no_urldecode_processor[] +* <> +endif::[] //# end::processors-list[] //# tag::processors-include[] @@ -204,5 +207,8 @@ endif::[] ifndef::no_translate_sid_processor[] include::{libbeat-processors-dir}/translate_sid/docs/translate_sid.asciidoc[] endif::[] +ifndef::no_urldecode_processor[] +include::{libbeat-processors-dir}/actions/docs/urldecode.asciidoc[] +endif::[] //# end::processors-include[] diff --git a/libbeat/processors/actions/docs/urldecode.asciidoc b/libbeat/processors/actions/docs/urldecode.asciidoc new file mode 100644 index 00000000000..ba675e922e7 --- /dev/null +++ b/libbeat/processors/actions/docs/urldecode.asciidoc @@ -0,0 +1,38 @@ +[[urldecode]] +=== URL Decode + +++++ +urldecode +++++ + +The `urldecode` processor specifies a list of fields to decode from URL encoded format. Under the `fields` +key, each entry contains a `from: old-key` and a `to: new-key` pair, where: + +* `from` is the original field name +* `to` is the target field name + +[source,yaml] +------- +processors: +- urldecode: + fields: + - from: "field1" + to: "field2" + ignore_missing: false + fail_on_error: true +------- + +In the example above: + +- field1 is decoded in field2 + +The `urldecode` processor has the following configuration settings: + +`ignore_missing`:: (Optional) If set to true, no error is logged in case a key +which should be URL-decoded is missing. Default is `false`. + +`fail_on_error`:: (Optional) If set to true, in case of an error the URL-decoding +of fields is stopped and the original event is returned. If set to false, decoding +continues also if an error happened during decoding. Default is `true`. + +See <> for a list of supported conditions. diff --git a/libbeat/processors/actions/urldecode.go b/libbeat/processors/actions/urldecode.go new file mode 100644 index 00000000000..5e1f04a1d25 --- /dev/null +++ b/libbeat/processors/actions/urldecode.go @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "fmt" + "net/url" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" +) + +type urlDecode struct { + config urlDecodeConfig + log *logp.Logger +} + +type urlDecodeConfig struct { + Fields []fromTo `config:"fields"` + IgnoreMissing bool `config:"ignore_missing"` + FailOnError bool `config:"fail_on_error"` +} + +func init() { + processors.RegisterPlugin("urldecode", + checks.ConfigChecked(NewURLDecode, + checks.RequireFields("fields"), + checks.AllowedFields("fields", "ignore_missing", "fail_on_error"))) + jsprocessor.RegisterPlugin("URLDecode", NewURLDecode) +} + +func NewURLDecode(c *common.Config) (processors.Processor, error) { + config := urlDecodeConfig{ + IgnoreMissing: false, + FailOnError: true, + } + + err := c.Unpack(&config) + if err != nil { + return nil, fmt.Errorf("failed to unpack the configuration of urldecode processor: %s", err) + } + + return &urlDecode{ + config: config, + log: logp.NewLogger("urldecode"), + }, nil + +} + +func (p *urlDecode) Run(event *beat.Event) (*beat.Event, error) { + var backup common.MapStr + if p.config.FailOnError { + backup = event.Fields.Clone() + } + + for _, field := range p.config.Fields { + err := p.decodeField(field.From, field.To, event) + if err != nil { + errMsg := fmt.Errorf("failed to decode fields in urldecode processor: %v", err) + p.log.Debug(errMsg.Error()) + if p.config.FailOnError { + event.Fields = backup + event.PutValue("error.message", errMsg.Error()) + return event, err + } + } + } + + return event, nil +} + +func (p *urlDecode) decodeField(from string, to string, event *beat.Event) error { + value, err := event.GetValue(from) + if err != nil { + if p.config.IgnoreMissing && errors.Cause(err) == common.ErrKeyNotFound { + return nil + } + return fmt.Errorf("could not fetch value for key: %s, Error: %v", from, err) + } + + encodedString, ok := value.(string) + if !ok { + return fmt.Errorf("invalid type for `from`, expecting a string received %T", value) + } + + decodedData, err := url.QueryUnescape(encodedString) + if err != nil { + return fmt.Errorf("error trying to URL-decode %s: %v", encodedString, err) + } + + target := to + // If to is empty + if to == "" || from == to { + target = from + } + + if _, err := event.PutValue(target, decodedData); err != nil { + return fmt.Errorf("could not put value: %s: %v, %v", decodedData, target, err) + } + + return nil +} + +func (p *urlDecode) String() string { + return "urldecode=" + fmt.Sprintf("%+v", p.config.Fields) +} diff --git a/libbeat/processors/actions/urldecode_test.go b/libbeat/processors/actions/urldecode_test.go new file mode 100644 index 00000000000..36fb84337ff --- /dev/null +++ b/libbeat/processors/actions/urldecode_test.go @@ -0,0 +1,214 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestURLDecode(t *testing.T) { + var testCases = []struct { + description string + config urlDecodeConfig + Input common.MapStr + Output common.MapStr + error bool + }{ + { + description: "simple field urldecode", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field1", To: "field2", + }}, + IgnoreMissing: false, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "correct%20data", + }, + Output: common.MapStr{ + "field1": "correct%20data", + "field2": "correct data", + }, + error: false, + }, + { + description: "simple multiple fields urldecode", + config: urlDecodeConfig{ + Fields: []fromTo{ + {From: "field1", To: "field2"}, + {From: "field3", To: "field4"}, + }, + IgnoreMissing: false, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "correct%20field1", + "field3": "correct%20field3", + }, + Output: common.MapStr{ + "field1": "correct%20field1", + "field2": "correct field1", + "field3": "correct%20field3", + "field4": "correct field3", + }, + error: false, + }, + { + description: "simple field urldecode To empty", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field1", To: "", + }}, + IgnoreMissing: false, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "correct%20data", + }, + Output: common.MapStr{ + "field1": "correct data", + }, + error: false, + }, + { + description: "simple field urldecode from and to equals", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field1", To: "field1", + }}, + IgnoreMissing: false, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "correct%20data", + }, + Output: common.MapStr{ + "field1": "correct data", + }, + error: false, + }, + { + description: "simple field bad data - fail on error", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field1", To: "field1", + }}, + IgnoreMissing: false, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "Hello G%ünter", + }, + Output: common.MapStr{ + "field1": "Hello G%ünter", + "error": common.MapStr{ + "message": "failed to decode fields in urldecode processor: error trying to URL-decode Hello G%ünter: invalid URL escape \"%ü\"", + }, + }, + error: true, + }, + { + description: "simple field bad data fail on error false", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field1", To: "field1", + }}, + IgnoreMissing: false, + FailOnError: false, + }, + Input: common.MapStr{ + "field1": "Hello G%ünter", + }, + Output: common.MapStr{ + "field1": "Hello G%ünter", + }, + error: false, + }, + { + description: "missing field", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field2", To: "field3", + }}, + IgnoreMissing: false, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "correct%20data", + }, + Output: common.MapStr{ + "field1": "correct%20data", + "error": common.MapStr{ + "message": "failed to decode fields in urldecode processor: could not fetch value for key: field2, Error: key not found", + }, + }, + error: true, + }, + { + description: "missing field ignore", + config: urlDecodeConfig{ + Fields: []fromTo{{ + From: "field2", To: "field3", + }}, + IgnoreMissing: true, + FailOnError: true, + }, + Input: common.MapStr{ + "field1": "correct%20data", + }, + Output: common.MapStr{ + "field1": "correct%20data", + }, + error: false, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.description, func(t *testing.T) { + //t.Parallel() + + f := &urlDecode{ + log: logp.NewLogger("urldecode"), + config: test.config, + } + + event := &beat.Event{ + Fields: test.Input, + } + + newEvent, err := f.Run(event) + if !test.error { + assert.Nil(t, err) + } else { + assert.NotNil(t, err) + } + + assert.Equal(t, test.Output, newEvent.Fields) + + }) + } + +} From e3e032c97397093b5a4946bf1eff6b14059991fe Mon Sep 17 00:00:00 2001 From: Chao Date: Mon, 6 Apr 2020 01:58:13 +0800 Subject: [PATCH 2/7] update reference yml files --- auditbeat/auditbeat.reference.yml | 10 +++++++++ filebeat/filebeat.reference.yml | 10 +++++++++ heartbeat/heartbeat.reference.yml | 10 +++++++++ journalbeat/journalbeat.reference.yml | 10 +++++++++ libbeat/_meta/config.reference.yml.tmpl | 10 +++++++++ metricbeat/metricbeat.reference.yml | 10 +++++++++ packetbeat/packetbeat.reference.yml | 10 +++++++++ winlogbeat/winlogbeat.reference.yml | 10 +++++++++ x-pack/auditbeat/auditbeat.reference.yml | 10 +++++++++ x-pack/filebeat/filebeat.reference.yml | 22 ++++++++++++++----- .../functionbeat/functionbeat.reference.yml | 10 +++++++++ x-pack/metricbeat/metricbeat.reference.yml | 10 +++++++++ x-pack/winlogbeat/winlogbeat.reference.yml | 10 +++++++++ 13 files changed, 136 insertions(+), 6 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 581e7bbe95d..4913b1f4780 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -378,6 +378,16 @@ auditbeat.modules: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 9649e8e1487..e22d16362bf 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1084,6 +1084,16 @@ filebeat.inputs: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index ba6e08ffe68..33863fae1a2 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -522,6 +522,16 @@ heartbeat.scheduler: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 0afd73547a8..7f5a33d5b9c 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -316,6 +316,16 @@ setup.template.settings: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl index 995e7bf3df4..65e6d8aa992 100644 --- a/libbeat/_meta/config.reference.yml.tmpl +++ b/libbeat/_meta/config.reference.yml.tmpl @@ -259,6 +259,16 @@ # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 7d0b9219e9a..aa46c1709f5 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1114,6 +1114,16 @@ metricbeat.modules: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 4c1e8f7fb50..2b036da5808 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -805,6 +805,16 @@ packetbeat.ignore_outgoing: false # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 23097261f64..d7e87ea2826 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -301,6 +301,16 @@ winlogbeat.event_logs: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index e21dd537f52..ab6c189656a 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -434,6 +434,16 @@ auditbeat.modules: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 838301aa98d..dc40882f29d 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -326,7 +326,7 @@ filebeat.modules: #------------------------------- Coredns Module ------------------------------- - module: coredns # Fileset for native deployment - log: + log: enabled: true # Set custom paths for the log files. If left empty, @@ -335,7 +335,7 @@ filebeat.modules: #----------------------------- Crowdstrike Module ----------------------------- - module: crowdstrike - + falcon: enabled: true @@ -380,7 +380,7 @@ filebeat.modules: #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy # Fileset for native deployment - log: + log: enabled: true # Set custom paths for the log files. If left empty, @@ -596,7 +596,7 @@ filebeat.modules: # URL of the MISP REST API #var.url - + # You can also pass SSL options. For example: #var.ssl: |- # { @@ -762,7 +762,7 @@ filebeat.modules: # URL of the Okta REST API #var.url - # Disable SSL verification + # Disable SSL verification #var.ssl: |- # { # "verification_mode": "none" @@ -893,7 +893,7 @@ filebeat.modules: http: enabled: true intel: - enabled: true + enabled: true irc: enabled: true kerberos: @@ -1729,6 +1729,16 @@ filebeat.inputs: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 7a85d3e805e..0e945a8dea2 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -644,6 +644,16 @@ functionbeat.provider.gcp.functions: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 462338ad399..e1f2bb16f0b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1522,6 +1522,16 @@ metricbeat.modules: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 435fc9b323a..e4265a3b3ec 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -304,6 +304,16 @@ winlogbeat.event_logs: # max_bytes: 1024 # fail_on_error: false # ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +#- urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true #============================= Elastic Cloud ================================== From e47bf4a3ff874d2a2ababdd9772ef5e972a29200 Mon Sep 17 00:00:00 2001 From: Chao Date: Mon, 6 Apr 2020 02:07:02 +0800 Subject: [PATCH 3/7] update doc with PR number --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 758ec3d8bd3..ea114076dc4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -175,7 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update supported versions of `redis` output. {pull}17198[17198] - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268] - Add optional regex based cid extractor to `add_kubernetes_metadata` processor. {pull}17360[17360] -- Add `urldecode` processor to for decoding URL encoded fields. +- Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505] *Auditbeat* From 924bd3746bfa7243247bfb01f95c007821d41e48 Mon Sep 17 00:00:00 2001 From: Chao Date: Mon, 6 Apr 2020 02:35:28 +0800 Subject: [PATCH 4/7] remove unexpected format --- x-pack/filebeat/filebeat.reference.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index dc40882f29d..dc47bdd07a8 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -326,7 +326,7 @@ filebeat.modules: #------------------------------- Coredns Module ------------------------------- - module: coredns # Fileset for native deployment - log: + log: enabled: true # Set custom paths for the log files. If left empty, @@ -335,7 +335,7 @@ filebeat.modules: #----------------------------- Crowdstrike Module ----------------------------- - module: crowdstrike - + falcon: enabled: true @@ -380,7 +380,7 @@ filebeat.modules: #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy # Fileset for native deployment - log: + log: enabled: true # Set custom paths for the log files. If left empty, @@ -596,7 +596,7 @@ filebeat.modules: # URL of the MISP REST API #var.url - + # You can also pass SSL options. For example: #var.ssl: |- # { @@ -762,7 +762,7 @@ filebeat.modules: # URL of the Okta REST API #var.url - # Disable SSL verification + # Disable SSL verification #var.ssl: |- # { # "verification_mode": "none" @@ -893,7 +893,7 @@ filebeat.modules: http: enabled: true intel: - enabled: true + enabled: true irc: enabled: true kerberos: From 627ae785e9283b6a14e816c9fb4c627451b43ec5 Mon Sep 17 00:00:00 2001 From: Chao Date: Tue, 7 Apr 2020 14:24:47 +0800 Subject: [PATCH 5/7] update from feedback --- libbeat/processors/actions/docs/urldecode.asciidoc | 6 +++--- libbeat/processors/actions/urldecode.go | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/libbeat/processors/actions/docs/urldecode.asciidoc b/libbeat/processors/actions/docs/urldecode.asciidoc index ba675e922e7..6a544749d2c 100644 --- a/libbeat/processors/actions/docs/urldecode.asciidoc +++ b/libbeat/processors/actions/docs/urldecode.asciidoc @@ -6,10 +6,10 @@ ++++ The `urldecode` processor specifies a list of fields to decode from URL encoded format. Under the `fields` -key, each entry contains a `from: old-key` and a `to: new-key` pair, where: +key, each entry contains a `from: source-field` and a `to: target-field` pair, where: -* `from` is the original field name -* `to` is the target field name +* `from` is the source field name +* `to` is the target field name (defaults to the `from` value) [source,yaml] ------- diff --git a/libbeat/processors/actions/urldecode.go b/libbeat/processors/actions/urldecode.go index 5e1f04a1d25..0f61b59986c 100644 --- a/libbeat/processors/actions/urldecode.go +++ b/libbeat/processors/actions/urldecode.go @@ -37,7 +37,7 @@ type urlDecode struct { } type urlDecodeConfig struct { - Fields []fromTo `config:"fields"` + Fields []fromTo `config:"fields" validate:"required"` IgnoreMissing bool `config:"ignore_missing"` FailOnError bool `config:"fail_on_error"` } @@ -56,8 +56,7 @@ func NewURLDecode(c *common.Config) (processors.Processor, error) { FailOnError: true, } - err := c.Unpack(&config) - if err != nil { + if err := c.Unpack(&config); err != nil { return nil, fmt.Errorf("failed to unpack the configuration of urldecode processor: %s", err) } @@ -110,8 +109,7 @@ func (p *urlDecode) decodeField(from string, to string, event *beat.Event) error } target := to - // If to is empty - if to == "" || from == to { + if to == "" { target = from } From 9e9457042d863b8a2fc0f3a0c1182ae8d998c1d9 Mon Sep 17 00:00:00 2001 From: Chao Date: Tue, 7 Apr 2020 14:48:02 +0800 Subject: [PATCH 6/7] move urldecode processor into its own package --- libbeat/cmd/instance/imports_common.go | 1 + libbeat/docs/processors-list.asciidoc | 2 +- .../{actions => urldecode}/docs/urldecode.asciidoc | 0 libbeat/processors/{actions => urldecode}/urldecode.go | 7 ++++++- .../processors/{actions => urldecode}/urldecode_test.go | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) rename libbeat/processors/{actions => urldecode}/docs/urldecode.asciidoc (100%) rename libbeat/processors/{actions => urldecode}/urldecode.go (97%) rename libbeat/processors/{actions => urldecode}/urldecode_test.go (99%) diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go index 5e993f6a2a0..a2b2569d61c 100644 --- a/libbeat/cmd/instance/imports_common.go +++ b/libbeat/cmd/instance/imports_common.go @@ -36,5 +36,6 @@ import ( _ "github.com/elastic/beats/v7/libbeat/processors/fingerprint" _ "github.com/elastic/beats/v7/libbeat/processors/registered_domain" _ "github.com/elastic/beats/v7/libbeat/processors/translate_sid" + _ "github.com/elastic/beats/v7/libbeat/processors/urldecode" _ "github.com/elastic/beats/v7/libbeat/publisher/includes" // Register publisher pipeline modules ) diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc index 5f78fc0dbea..169657aecc4 100644 --- a/libbeat/docs/processors-list.asciidoc +++ b/libbeat/docs/processors-list.asciidoc @@ -208,7 +208,7 @@ ifndef::no_translate_sid_processor[] include::{libbeat-processors-dir}/translate_sid/docs/translate_sid.asciidoc[] endif::[] ifndef::no_urldecode_processor[] -include::{libbeat-processors-dir}/actions/docs/urldecode.asciidoc[] +include::{libbeat-processors-dir}/urldecode/docs/urldecode.asciidoc[] endif::[] //# end::processors-include[] diff --git a/libbeat/processors/actions/docs/urldecode.asciidoc b/libbeat/processors/urldecode/docs/urldecode.asciidoc similarity index 100% rename from libbeat/processors/actions/docs/urldecode.asciidoc rename to libbeat/processors/urldecode/docs/urldecode.asciidoc diff --git a/libbeat/processors/actions/urldecode.go b/libbeat/processors/urldecode/urldecode.go similarity index 97% rename from libbeat/processors/actions/urldecode.go rename to libbeat/processors/urldecode/urldecode.go index 0f61b59986c..96b155b8356 100644 --- a/libbeat/processors/actions/urldecode.go +++ b/libbeat/processors/urldecode/urldecode.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package actions +package urldecode import ( "fmt" @@ -42,6 +42,11 @@ type urlDecodeConfig struct { FailOnError bool `config:"fail_on_error"` } +type fromTo struct { + From string `config:"from"` + To string `config:"to"` +} + func init() { processors.RegisterPlugin("urldecode", checks.ConfigChecked(NewURLDecode, diff --git a/libbeat/processors/actions/urldecode_test.go b/libbeat/processors/urldecode/urldecode_test.go similarity index 99% rename from libbeat/processors/actions/urldecode_test.go rename to libbeat/processors/urldecode/urldecode_test.go index 36fb84337ff..bf10244965d 100644 --- a/libbeat/processors/actions/urldecode_test.go +++ b/libbeat/processors/urldecode/urldecode_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package actions +package urldecode import ( "testing" From 0347640dadb6e5fc7da04abd81b54b529b3fa738 Mon Sep 17 00:00:00 2001 From: Chao Date: Tue, 7 Apr 2020 17:49:27 +0800 Subject: [PATCH 7/7] update from feedback --- libbeat/processors/urldecode/urldecode.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libbeat/processors/urldecode/urldecode.go b/libbeat/processors/urldecode/urldecode.go index 96b155b8356..6fc9cb8386d 100644 --- a/libbeat/processors/urldecode/urldecode.go +++ b/libbeat/processors/urldecode/urldecode.go @@ -43,19 +43,19 @@ type urlDecodeConfig struct { } type fromTo struct { - From string `config:"from"` + From string `config:"from" validate:"required"` To string `config:"to"` } func init() { processors.RegisterPlugin("urldecode", - checks.ConfigChecked(NewURLDecode, + checks.ConfigChecked(New, checks.RequireFields("fields"), checks.AllowedFields("fields", "ignore_missing", "fail_on_error"))) - jsprocessor.RegisterPlugin("URLDecode", NewURLDecode) + jsprocessor.RegisterPlugin("URLDecode", New) } -func NewURLDecode(c *common.Config) (processors.Processor, error) { +func New(c *common.Config) (processors.Processor, error) { config := urlDecodeConfig{ IgnoreMissing: false, FailOnError: true,