diff --git a/docs/config-schema.md b/docs/config-schema.md index d71c876f2f..fe425f8f75 100644 --- a/docs/config-schema.md +++ b/docs/config-schema.md @@ -154,6 +154,7 @@ The **nested** `writer` config object has the following fields: | `logDatapoints` | no | bool | If the log level is set to `debug` and this is true, all datapoints generated by the agent will be logged. (**default:** `false`) | | `logEvents` | no | bool | The analogue of `logDatapoints` for events. (**default:** `false`) | | `logTraceSpans` | no | bool | The analogue of `logDatapoints` for trace spans. (**default:** `false`) | +| `logTraceSpansFailedToShip` | no | bool | If `true`, traces and spans which weren't successfully received by the backend, will be logged as json (**default:** `false`) | | `logDimensionUpdates` | no | bool | If `true`, dimension updates will be logged at the INFO level. (**default:** `false`) | | `logDroppedDatapoints` | no | bool | If true, and the log level is `debug`, filtered out datapoints will be logged. (**default:** `false`) | | `addGlobalDimensionsAsSpanTags` | no | bool | If true, the dimensions specified in the top-level `globalDimensions` configuration will be added to the tag set of all spans that are emitted by the writer. If this is false, only the "host id" dimensions such as `host`, `AwsUniqueId`, etc. are added to the span tags. (**default:** `false`) | @@ -428,6 +429,7 @@ where applicable: logDatapoints: false logEvents: false logTraceSpans: false + logTraceSpansFailedToShip: false logDimensionUpdates: false logDroppedDatapoints: false addGlobalDimensionsAsSpanTags: false diff --git a/pkg/core/config/writer.go b/pkg/core/config/writer.go index e9b0adcbce..a92c7145c6 100644 --- a/pkg/core/config/writer.go +++ b/pkg/core/config/writer.go @@ -71,6 +71,9 @@ type WriterConfig struct { LogEvents bool `yaml:"logEvents"` // The analogue of `logDatapoints` for trace spans. LogTraceSpans bool `yaml:"logTraceSpans"` + // If `true`, traces and spans which weren't successfully received by the + // backend, will be logged as json + LogTraceSpansFailedToShip bool `yaml:"logTraceSpansFailedToShip"` // If `true`, dimension updates will be logged at the INFO level. LogDimensionUpdates bool `yaml:"logDimensionUpdates"` // If true, and the log level is `debug`, filtered out datapoints will be diff --git a/pkg/core/writer/signalfx/spans.go b/pkg/core/writer/signalfx/spans.go index de1bcec163..bb8e498d15 100644 --- a/pkg/core/writer/signalfx/spans.go +++ b/pkg/core/writer/signalfx/spans.go @@ -21,9 +21,21 @@ func (sw *Writer) sendSpans(ctx context.Context, spans []*trace.Span) error { // This sends synchonously err := sw.client.AddSpans(context.Background(), spans) if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Error shipping spans to SignalFx") + var meta log.Fields + if sw.conf.LogTraceSpansFailedToShip { + jsonEncodedSpans, _ := json.Marshal(spans) + meta = log.Fields{ + "error": err, + "payload": string(jsonEncodedSpans), + } + } else { + meta = log.Fields{ + "error": err, + } + } + + log.WithFields(meta).Error("Error shipping spans to SignalFx") + // If there is an error sending spans then just forget about them. return err } diff --git a/selfdescribe.json b/selfdescribe.json index 189fa8f5a5..e8b0628fa6 100644 --- a/selfdescribe.json +++ b/selfdescribe.json @@ -58322,6 +58322,14 @@ "type": "bool", "elementKind": "" }, + { + "yamlName": "logTraceSpansFailedToShip", + "doc": "If `true`, traces and spans which weren't successfully received by the backend, will be logged as json", + "default": false, + "required": false, + "type": "bool", + "elementKind": "" + }, { "yamlName": "logDimensionUpdates", "doc": "If `true`, dimension updates will be logged at the INFO level.",