From 8e8a730054690500909cca92c6ef63ec70e7935f Mon Sep 17 00:00:00 2001 From: Jakub Malinowski Date: Wed, 22 Sep 2021 15:08:31 +0200 Subject: [PATCH 1/4] writer: new option to log rejected spans --- docs/config-schema.md | 111 +++++++++++++++--------------- pkg/core/config/writer.go | 3 + pkg/core/writer/signalfx/spans.go | 18 ++++- 3 files changed, 74 insertions(+), 58 deletions(-) diff --git a/docs/config-schema.md b/docs/config-schema.md index d71c876f2f..01cc2037f8 100644 --- a/docs/config-schema.md +++ b/docs/config-schema.md @@ -154,6 +154,7 @@ The **nested** `writer` config object has the following fields: | `logDatapoints` | no | bool | If the log level is set to `debug` and this is true, all datapoints generated by the agent will be logged. (**default:** `false`) | | `logEvents` | no | bool | The analogue of `logDatapoints` for events. (**default:** `false`) | | `logTraceSpans` | no | bool | The analogue of `logDatapoints` for trace spans. (**default:** `false`) | +| `LogTraceSpansFailedToShip` | no | bool | If `true`, traces and spans which weren't successfully received by the backend, will be logged as json. (**default:** `false`) | | `logDimensionUpdates` | no | bool | If `true`, dimension updates will be logged at the INFO level. (**default:** `false`) | | `logDroppedDatapoints` | no | bool | If true, and the log level is `debug`, filtered out datapoints will be logged. (**default:** `false`) | | `addGlobalDimensionsAsSpanTags` | no | bool | If true, the dimensions specified in the top-level `globalDimensions` configuration will be added to the tag set of all spans that are emitted by the writer. If this is false, only the "host id" dimensions such as `host`, `AwsUniqueId`, etc. are added to the span tags. (**default:** `false`) | @@ -394,25 +395,25 @@ where applicable: ```yaml - signalFxAccessToken: - ingestUrl: - eventEndpointUrl: - traceEndpointUrl: - apiUrl: + signalFxAccessToken: + ingestUrl: + eventEndpointUrl: + traceEndpointUrl: + apiUrl: signalFxRealm: "us0" - hostname: - useFullyQualifiedHost: + hostname: + useFullyQualifiedHost: disableHostDimensions: false intervalSeconds: 10 cloudMetadataTimeout: "2s" - globalDimensions: - globalSpanTags: - cluster: + globalDimensions: + globalSpanTags: + cluster: syncClusterOnHostDimension: false validateDiscoveryRules: false observers: [] monitors: [] - writer: + writer: datapointMaxBatchSize: 1000 maxDatapointsBuffered: 25000 traceSpanMaxBatchSize: 1000 @@ -437,26 +438,26 @@ where applicable: traceHostCorrelationMetricsInterval: "1m" traceHostCorrelationMaxRequestRetries: 2 maxTraceSpansInFlight: 100000 - splunk: + splunk: enabled: false - url: - token: - source: - sourceType: - index: - eventsIndex: - eventsSource: - eventsSourceType: + url: + token: + source: + sourceType: + index: + eventsIndex: + eventsSource: + eventsSourceType: skipTLSVerify: false maxBuffered: 0 maxRequests: 0 maxBatchSize: 0 signalFxEnabled: true - extraHeaders: - logging: + extraHeaders: + logging: level: "info" format: "text" - collectd: + collectd: disableCollectd: false timeout: 40 readThreads: 5 @@ -477,45 +478,45 @@ where applicable: profiling: false profilingHost: "127.0.0.1" profilingPort: 6060 - bundleDir: - scratch: - configSources: + bundleDir: + scratch: + configSources: watch: true remoteWatch: true - file: + file: pollRateSeconds: 5 - zookeeper: + zookeeper: endpoints: [] timeoutSeconds: 10 - etcd2: + etcd2: endpoints: [] - username: - password: - consul: - endpoint: - username: - password: - token: - datacenter: - vault: - vaultAddr: - vaultToken: + username: + password: + consul: + endpoint: + username: + password: + token: + datacenter: + vault: + vaultAddr: + vaultToken: kvV2PollInterval: "60s" - authMethod: - iam: - awsAccessKeyId: - awsSecretAccessKey: - awsSecurityToken: - headerValue: - mount: - role: - gcp: - role: - mount: - credentials: - jwt_exp: - service_account: - project: + authMethod: + iam: + awsAccessKeyId: + awsSecretAccessKey: + awsSecurityToken: + headerValue: + mount: + role: + gcp: + role: + mount: + credentials: + jwt_exp: + service_account: + project: procPath: "/proc" etcPath: "/etc" varPath: "/var" diff --git a/pkg/core/config/writer.go b/pkg/core/config/writer.go index e9b0adcbce..a92c7145c6 100644 --- a/pkg/core/config/writer.go +++ b/pkg/core/config/writer.go @@ -71,6 +71,9 @@ type WriterConfig struct { LogEvents bool `yaml:"logEvents"` // The analogue of `logDatapoints` for trace spans. LogTraceSpans bool `yaml:"logTraceSpans"` + // If `true`, traces and spans which weren't successfully received by the + // backend, will be logged as json + LogTraceSpansFailedToShip bool `yaml:"logTraceSpansFailedToShip"` // If `true`, dimension updates will be logged at the INFO level. LogDimensionUpdates bool `yaml:"logDimensionUpdates"` // If true, and the log level is `debug`, filtered out datapoints will be diff --git a/pkg/core/writer/signalfx/spans.go b/pkg/core/writer/signalfx/spans.go index de1bcec163..c69fd7eefc 100644 --- a/pkg/core/writer/signalfx/spans.go +++ b/pkg/core/writer/signalfx/spans.go @@ -21,9 +21,21 @@ func (sw *Writer) sendSpans(ctx context.Context, spans []*trace.Span) error { // This sends synchonously err := sw.client.AddSpans(context.Background(), spans) if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Error shipping spans to SignalFx") + var meta log.Fields + if sw.conf.LogTraceSpansFailedToShip { + jsonEncodedSpans, _ := json.Marshal(spans) + meta = log.Fields{ + "error": err, + "payload": jsonEncodedSpans, + } + } else { + meta = log.Fields{ + "error": err, + } + } + + log.WithFields(meta).Error("Error shipping spans to SignalFx") + // If there is an error sending spans then just forget about them. return err } From 1422904f4c313c9b1798c6cad1aa48133f065224 Mon Sep 17 00:00:00 2001 From: Jakub Malinowski Date: Wed, 22 Sep 2021 17:10:19 +0200 Subject: [PATCH 2/4] chore: regenerate docs --- docs/config-schema.md | 3 ++- selfdescribe.json | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/config-schema.md b/docs/config-schema.md index 01cc2037f8..a6fdcca9d8 100644 --- a/docs/config-schema.md +++ b/docs/config-schema.md @@ -154,7 +154,7 @@ The **nested** `writer` config object has the following fields: | `logDatapoints` | no | bool | If the log level is set to `debug` and this is true, all datapoints generated by the agent will be logged. (**default:** `false`) | | `logEvents` | no | bool | The analogue of `logDatapoints` for events. (**default:** `false`) | | `logTraceSpans` | no | bool | The analogue of `logDatapoints` for trace spans. (**default:** `false`) | -| `LogTraceSpansFailedToShip` | no | bool | If `true`, traces and spans which weren't successfully received by the backend, will be logged as json. (**default:** `false`) | +| `logTraceSpansFailedToShip` | no | bool | If `true`, traces and spans which weren't successfully received by the backend, will be logged as json (**default:** `false`) | | `logDimensionUpdates` | no | bool | If `true`, dimension updates will be logged at the INFO level. (**default:** `false`) | | `logDroppedDatapoints` | no | bool | If true, and the log level is `debug`, filtered out datapoints will be logged. (**default:** `false`) | | `addGlobalDimensionsAsSpanTags` | no | bool | If true, the dimensions specified in the top-level `globalDimensions` configuration will be added to the tag set of all spans that are emitted by the writer. If this is false, only the "host id" dimensions such as `host`, `AwsUniqueId`, etc. are added to the span tags. (**default:** `false`) | @@ -429,6 +429,7 @@ where applicable: logDatapoints: false logEvents: false logTraceSpans: false + logTraceSpansFailedToShip: false logDimensionUpdates: false logDroppedDatapoints: false addGlobalDimensionsAsSpanTags: false diff --git a/selfdescribe.json b/selfdescribe.json index 189fa8f5a5..e8b0628fa6 100644 --- a/selfdescribe.json +++ b/selfdescribe.json @@ -58322,6 +58322,14 @@ "type": "bool", "elementKind": "" }, + { + "yamlName": "logTraceSpansFailedToShip", + "doc": "If `true`, traces and spans which weren't successfully received by the backend, will be logged as json", + "default": false, + "required": false, + "type": "bool", + "elementKind": "" + }, { "yamlName": "logDimensionUpdates", "doc": "If `true`, dimension updates will be logged at the INFO level.", From 710e0a2c8a15fe58b437e40eaa230a28cbf99983 Mon Sep 17 00:00:00 2001 From: Jakub Malinowski Date: Wed, 22 Sep 2021 20:11:36 +0200 Subject: [PATCH 3/4] fix: white-space in docs --- docs/config-schema.md | 110 +++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/docs/config-schema.md b/docs/config-schema.md index a6fdcca9d8..fe425f8f75 100644 --- a/docs/config-schema.md +++ b/docs/config-schema.md @@ -395,25 +395,25 @@ where applicable: ```yaml - signalFxAccessToken: - ingestUrl: - eventEndpointUrl: - traceEndpointUrl: - apiUrl: + signalFxAccessToken: + ingestUrl: + eventEndpointUrl: + traceEndpointUrl: + apiUrl: signalFxRealm: "us0" - hostname: - useFullyQualifiedHost: + hostname: + useFullyQualifiedHost: disableHostDimensions: false intervalSeconds: 10 cloudMetadataTimeout: "2s" - globalDimensions: - globalSpanTags: - cluster: + globalDimensions: + globalSpanTags: + cluster: syncClusterOnHostDimension: false validateDiscoveryRules: false observers: [] monitors: [] - writer: + writer: datapointMaxBatchSize: 1000 maxDatapointsBuffered: 25000 traceSpanMaxBatchSize: 1000 @@ -439,26 +439,26 @@ where applicable: traceHostCorrelationMetricsInterval: "1m" traceHostCorrelationMaxRequestRetries: 2 maxTraceSpansInFlight: 100000 - splunk: + splunk: enabled: false - url: - token: - source: - sourceType: - index: - eventsIndex: - eventsSource: - eventsSourceType: + url: + token: + source: + sourceType: + index: + eventsIndex: + eventsSource: + eventsSourceType: skipTLSVerify: false maxBuffered: 0 maxRequests: 0 maxBatchSize: 0 signalFxEnabled: true - extraHeaders: - logging: + extraHeaders: + logging: level: "info" format: "text" - collectd: + collectd: disableCollectd: false timeout: 40 readThreads: 5 @@ -479,45 +479,45 @@ where applicable: profiling: false profilingHost: "127.0.0.1" profilingPort: 6060 - bundleDir: - scratch: - configSources: + bundleDir: + scratch: + configSources: watch: true remoteWatch: true - file: + file: pollRateSeconds: 5 - zookeeper: + zookeeper: endpoints: [] timeoutSeconds: 10 - etcd2: + etcd2: endpoints: [] - username: - password: - consul: - endpoint: - username: - password: - token: - datacenter: - vault: - vaultAddr: - vaultToken: + username: + password: + consul: + endpoint: + username: + password: + token: + datacenter: + vault: + vaultAddr: + vaultToken: kvV2PollInterval: "60s" - authMethod: - iam: - awsAccessKeyId: - awsSecretAccessKey: - awsSecurityToken: - headerValue: - mount: - role: - gcp: - role: - mount: - credentials: - jwt_exp: - service_account: - project: + authMethod: + iam: + awsAccessKeyId: + awsSecretAccessKey: + awsSecurityToken: + headerValue: + mount: + role: + gcp: + role: + mount: + credentials: + jwt_exp: + service_account: + project: procPath: "/proc" etcPath: "/etc" varPath: "/var" From 2a9c2f7e5eae9d71371358ec48a258814cbbdb56 Mon Sep 17 00:00:00 2001 From: Jakub Malinowski Date: Wed, 22 Sep 2021 20:51:31 +0200 Subject: [PATCH 4/4] fix: format logged spans --- pkg/core/writer/signalfx/spans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/writer/signalfx/spans.go b/pkg/core/writer/signalfx/spans.go index c69fd7eefc..bb8e498d15 100644 --- a/pkg/core/writer/signalfx/spans.go +++ b/pkg/core/writer/signalfx/spans.go @@ -26,7 +26,7 @@ func (sw *Writer) sendSpans(ctx context.Context, spans []*trace.Span) error { jsonEncodedSpans, _ := json.Marshal(spans) meta = log.Fields{ "error": err, - "payload": jsonEncodedSpans, + "payload": string(jsonEncodedSpans), } } else { meta = log.Fields{