From c5aba59d513020b93ab5b2b25dd9c8b4d3dd42b0 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 11:57:04 +0200 Subject: [PATCH 01/12] remove event.received from app_logs --- .../app_logs/elasticsearch/ingest_pipeline/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml index 1f20b44c199..00efe090955 100644 --- a/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml @@ -9,3 +9,8 @@ processors: name: observer_ids - pipeline: name: ecs_version + - remove: + field: event.received + ignore_missing: true + ignore_failure: true + description: The field is used for internal statistics but does not need to be indexed. From 050b238fcfd33bbfe3af4f729a10d8de4901818d Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 11:57:22 +0200 Subject: [PATCH 02/12] remove event.received from app_metrics --- .../app_metrics/elasticsearch/ingest_pipeline/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml index 835e4f6bb4c..054f0b3da66 100644 --- a/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml @@ -15,3 +15,8 @@ processors: name: client_geoip - pipeline: name: set_metrics + - remove: + field: event.received + ignore_missing: true + ignore_failure: true + description: The field is used for internal statistics but does not need to be indexed. From f1251680a1fc5840a2b7304a2608efdc732cff31 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 11:57:33 +0200 Subject: [PATCH 03/12] remove event.received from error_logs --- .../error_logs/elasticsearch/ingest_pipeline/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml index 1433914ef7a..cb703af2181 100644 --- a/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml @@ -20,3 +20,8 @@ processors: if: ctx.error?.log?.message != null field: error.grouping_name copy_from: error.log.message + - remove: + field: event.received + ignore_missing: true + ignore_failure: true + description: The field is used for internal statistics but does not need to be indexed. From 663e33385718997aea1efa0e61cbb18f1063de2d Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 11:57:41 +0200 Subject: [PATCH 04/12] remove event.received from internal_metrics --- .../elasticsearch/ingest_pipeline/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml index 4efa6d388e8..3c29bc1a7e9 100644 --- a/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml @@ -22,3 +22,8 @@ processors: - remove: field: _dynamic_templates ignore_missing: true + - remove: + field: event.received + ignore_missing: true + ignore_failure: true + description: The field is used for internal statistics but does not need to be indexed. From f58d6c4800c948a5ae1022cec8f2fedbce898250 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 11:57:50 +0200 Subject: [PATCH 05/12] remove event.received from rum_traces --- .../rum_traces/elasticsearch/ingest_pipeline/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml index 7bcf6be7437..8d40d4c4cce 100644 --- a/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml @@ -38,4 +38,9 @@ processors: - service.framework ignore_missing: true ignore_failure: true + - remove: + field: event.received + ignore_missing: true + ignore_failure: true + description: The field is used for internal statistics but does not need to be indexed. From aa5e4ea9d10224b1101e04f95548f89059d88be8 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 11:57:58 +0200 Subject: [PATCH 06/12] remove event.received from traces --- .../traces/elasticsearch/ingest_pipeline/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml index 757fd71d9b0..a289a9b23ba 100644 --- a/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml @@ -35,3 +35,8 @@ processors: value: 0 ignore_failure: true # end of event.success_count logic + - remove: + field: event.received + ignore_missing: true + ignore_failure: true + description: The field is used for internal statistics but does not need to be indexed. From cbd839440e2b5c88f2356ba2a2e60307e8b6e0f8 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 12:09:06 +0200 Subject: [PATCH 07/12] update changelog --- apmpackage/apm/changelog.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml index 654ee5d0668..2bcb00bb5ab 100644 --- a/apmpackage/apm/changelog.yml +++ b/apmpackage/apm/changelog.yml @@ -12,6 +12,9 @@ - description: Remove unused `processor.event` field from logs data streams type: enhancement link: https://github.com/elastic/apm-server/pull/11494 + - description: Remove `event.received` fields from logs, metrics and traces data streams + type: bugfix + link: - version: "8.10.0" changes: - description: Add permissions to reroute to dedicated datasets for logs, metrics and traces From aac65811d81bd1b7c52ae4854c452eda1147748f Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 14:36:45 +0200 Subject: [PATCH 08/12] add PR link to changelog --- apmpackage/apm/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml index 2bcb00bb5ab..0684483abf3 100644 --- a/apmpackage/apm/changelog.yml +++ b/apmpackage/apm/changelog.yml @@ -14,7 +14,7 @@ link: https://github.com/elastic/apm-server/pull/11494 - description: Remove `event.received` fields from logs, metrics and traces data streams type: bugfix - link: + link: https://github.com/elastic/apm-server/pull/11602 - version: "8.10.0" changes: - description: Add permissions to reroute to dedicated datasets for logs, metrics and traces From 8ac04bdc9f8da3d61810e34e2dcfcd240e41ff6a Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 15:10:46 +0200 Subject: [PATCH 09/12] remove removeEventReceivedBatchProcessor The same functionality is not handled in ingest pipelines --- internal/beater/beater.go | 4 ---- internal/beater/processors.go | 11 ----------- 2 files changed, 15 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 693445648bf..1477b700627 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -509,10 +509,6 @@ func (s *Runner) Run(ctx context.Context) error { modelpb.ProcessBatchFunc(rateLimitBatchProcessor), modelpb.ProcessBatchFunc(authorizeEventIngestProcessor), - // Add a model processor that removes `event.received`, which is added by - // apm-data, but which we don't yet map. - modelpb.ProcessBatchFunc(removeEventReceivedBatchProcessor), - // Pre-process events before they are sent to the final processors for // aggregation, sampling, and indexing. modelprocessor.SetHostHostname{}, diff --git a/internal/beater/processors.go b/internal/beater/processors.go index 74d57022085..c81d9c6c4a8 100644 --- a/internal/beater/processors.go +++ b/internal/beater/processors.go @@ -85,17 +85,6 @@ func newObserverBatchProcessor() modelpb.ProcessBatchFunc { } } -// TODO remove this once we have added `event.received` to the -// data stream mappings, in 8.10. -func removeEventReceivedBatchProcessor(ctx context.Context, batch *modelpb.Batch) error { - for _, event := range *batch { - if event.Event != nil { - event.Event.Received = 0 - } - } - return nil -} - func newDocappenderBatchProcessor(a *docappender.Appender) modelpb.ProcessBatchFunc { var pool sync.Pool pool.New = func() any { From 20051e9311afe6a30b853bc76d3d021f4d1f56b0 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 15:37:45 +0200 Subject: [PATCH 10/12] add remove_event_received to genpackage --- apmpackage/cmd/genpackage/pipelines.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/apmpackage/cmd/genpackage/pipelines.go b/apmpackage/cmd/genpackage/pipelines.go index 3387d9830de..beb680dabf5 100644 --- a/apmpackage/cmd/genpackage/pipelines.go +++ b/apmpackage/cmd/genpackage/pipelines.go @@ -36,14 +36,15 @@ import ( // - ... func getCommonPipeline(name string, version *version.V) []map[string]interface{} { commonPipelines := map[string][]map[string]interface{}{ - "observer_version": getObserverVersionPipeline(version), - "observer_ids": observerIDsPipeline, - "ecs_version": ecsVersionPipeline, - "user_agent": userAgentPipeline, - "process_ppid": processPpidPipeline, - "client_geoip": clientGeoIPPipeline, - "event_duration": eventDurationPipeline, - "set_metrics": setMetricsPipeline, + "observer_version": getObserverVersionPipeline(version), + "observer_ids": observerIDsPipeline, + "ecs_version": ecsVersionPipeline, + "user_agent": userAgentPipeline, + "process_ppid": processPpidPipeline, + "client_geoip": clientGeoIPPipeline, + "event_duration": eventDurationPipeline, + "set_metrics": setMetricsPipeline, + "remove_event_received": removeEventReceivedPipeline, } return commonPipelines[name] } @@ -144,6 +145,15 @@ var clientGeoIPPipeline = []map[string]interface{}{{ }, }} +var removeEventReceivedPipeline = []map[string]interface{}{{ + "remove": map[string]interface{}{ + "field": "event.received", + "ignore_missing": true, + "ignore_failure": true, + "description": "The field is used for internal statistics but does not need to be indexed.", + }, +}} + // This pipeline translates `event.duration` (defaulting to zero if not // found) to `transaction.duration.us` or `span.duration.us` depending on // the event type, and then removes `event.duration`. Older versions of From 8e51d6bf0087390db3b78b4de8478b76569be46b Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Thu, 7 Sep 2023 15:38:33 +0200 Subject: [PATCH 11/12] use shared pipeline Replace previous implementation with the shared pipeline --- .../app_logs/elasticsearch/ingest_pipeline/default.yml | 7 ++----- .../app_metrics/elasticsearch/ingest_pipeline/default.yml | 7 ++----- .../error_logs/elasticsearch/ingest_pipeline/default.yml | 7 ++----- .../elasticsearch/ingest_pipeline/default.yml | 7 ++----- .../rum_traces/elasticsearch/ingest_pipeline/default.yml | 8 ++------ .../traces/elasticsearch/ingest_pipeline/default.yml | 7 ++----- 6 files changed, 12 insertions(+), 31 deletions(-) diff --git a/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml index 00efe090955..4a750b565e6 100644 --- a/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml @@ -9,8 +9,5 @@ processors: name: observer_ids - pipeline: name: ecs_version - - remove: - field: event.received - ignore_missing: true - ignore_failure: true - description: The field is used for internal statistics but does not need to be indexed. + - pipeline: + name: remove_event_received diff --git a/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml index 054f0b3da66..52bf97c5866 100644 --- a/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml @@ -15,8 +15,5 @@ processors: name: client_geoip - pipeline: name: set_metrics - - remove: - field: event.received - ignore_missing: true - ignore_failure: true - description: The field is used for internal statistics but does not need to be indexed. + - pipeline: + name: remove_event_received diff --git a/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml index cb703af2181..7f39701fb60 100644 --- a/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml @@ -20,8 +20,5 @@ processors: if: ctx.error?.log?.message != null field: error.grouping_name copy_from: error.log.message - - remove: - field: event.received - ignore_missing: true - ignore_failure: true - description: The field is used for internal statistics but does not need to be indexed. + - pipeline: + name: remove_event_received diff --git a/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml index 3c29bc1a7e9..3e0ba52f1ff 100644 --- a/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml @@ -22,8 +22,5 @@ processors: - remove: field: _dynamic_templates ignore_missing: true - - remove: - field: event.received - ignore_missing: true - ignore_failure: true - description: The field is used for internal statistics but does not need to be indexed. + - pipeline: + name: remove_event_received diff --git a/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml index 8d40d4c4cce..27a975e403a 100644 --- a/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml @@ -38,9 +38,5 @@ processors: - service.framework ignore_missing: true ignore_failure: true - - remove: - field: event.received - ignore_missing: true - ignore_failure: true - description: The field is used for internal statistics but does not need to be indexed. - + - pipeline: + name: remove_event_received diff --git a/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml index a289a9b23ba..5396a5bdb1d 100644 --- a/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml @@ -35,8 +35,5 @@ processors: value: 0 ignore_failure: true # end of event.success_count logic - - remove: - field: event.received - ignore_missing: true - ignore_failure: true - description: The field is used for internal statistics but does not need to be indexed. + - pipeline: + name: remove_event_received From cee297b83c3485d41e51e86f24c714f89af522fd Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Fri, 8 Sep 2023 09:59:19 +0200 Subject: [PATCH 12/12] changelog: is enhancement --- apmpackage/apm/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml index 0684483abf3..33c4ee5999c 100644 --- a/apmpackage/apm/changelog.yml +++ b/apmpackage/apm/changelog.yml @@ -13,7 +13,7 @@ type: enhancement link: https://github.com/elastic/apm-server/pull/11494 - description: Remove `event.received` fields from logs, metrics and traces data streams - type: bugfix + type: enhancement link: https://github.com/elastic/apm-server/pull/11602 - version: "8.10.0" changes: