x-pack/filebeat/input/internal/httplog: improve req/resp logging (#39455

) Attempt to log the request and response bodies and other details even when copying the body has been reported to have failed. (cherry picked from commit 3efb1e8) # Conflicts: # x-pack/filebeat/input/internal/httplog/roundtripper.go
elastic · May 9, 2024 · d6cdf29 · d6cdf29
1 parent d7e7efb
commit d6cdf29
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 0 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -86,6 +86,60 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Filebeat*
 
+- Adding Saved Object name field to Kibana audit logs {pull}38307[38307]
+- Update SQL input documentation regarding Oracle DSNs {pull}37590[37590]
+- add documentation for decode_xml_wineventlog processor field mappings.  {pull}32456[32456]
+- httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412]
+- Add cloudflare R2 to provider list in AWS S3 input. {pull}32620[32620]
+- Add support for single string containing multiple relation-types in getRFC5988Link. {pull}32811[32811]
+- Added separation of transform context object inside httpjson. Introduced new clause `.parent_last_response.*` {pull}33499[33499]
+- Added metric `sqs_messages_waiting_gauge` for aws-s3 input. {pull}34488[34488]
+- Add nginx.ingress_controller.upstream.ip to related.ip {issue}34645[34645] {pull}34672[34672]
+- Add unix socket log parsing for nginx ingress_controller {pull}34732[34732]
+- Added metric `sqs_worker_utilization` for aws-s3 input. {pull}34793[34793]
+- Add MySQL authentication message parsing and `related.ip` and `related.user` fields {pull}34810[34810]
+- Add nginx ingress_controller parsing if one of upstreams fails to return response {pull}34787[34787]
+- Add oracle authentication messages parsing {pull}35127[35127]
+- Add `clean_session` configuration setting for MQTT input.  {pull}35806[16204]
+- Add support for a simplified input configuraton when running under Elastic-Agent {pull}36390[36390]
+- Added support for Okta OAuth2 provider in the CEL input. {issue}36336[36336] {pull}36521[36521]
+- Added support for new features & removed partial save mechanism in the Azure Blob Storage input. {issue}35126[35126] {pull}36690[36690]
+- Added support for new features and removed partial save mechanism in the GCS input. {issue}35847[35847] {pull}36713[36713]
+- Use filestream input with file_identity.fingerprint as default for hints autodiscover. {issue}35984[35984] {pull}36950[36950]
+- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999]
+- Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128]
+- Made GCS input GA and updated docs accordingly. {pull}37127[37127]
+- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160]
+- Prevent CEL input from re-entering the eval loop when an evaluation failed. {pull}37161[37161]
+- Update CEL extensions library to v1.7.0. {pull}37172[37172]
+- Add support for complete URL replacement in HTTPJSON chain steps. {pull}37486[37486]
+- Add support for user-defined query selection in EntraID entity analytics provider. {pull}37653[37653]
+- Update CEL extensions library to v1.8.0 to provide runtime error location reporting. {issue}37304[37304] {pull}37718[37718]
+- Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682]
+- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714]
+- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772]
+- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836]
+- Added experimental version of the Websocket Input. {pull}37774[37774]
+- Add support for PEM-based Okta auth in CEL. {pull}37813[37813]
+- Add Salesforce input. {pull}37331[37331]
+- Add ETW input. {pull}36915[36915]
+- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971]
+- Add logging for cache processor file reads and writes. {pull}38052[38052]
+- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738]
+- Support VPC endpoint for aws-s3 input SQS queue url. {pull}38189[38189]
+- Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237]
+- Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738]
+- Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193]
+- Parse more fields from Elasticsearch slowlogs {pull}38295[38295]
+- Update CEL mito extensions to v1.10.0 to add base64 decode functions. {pull}38504[38504]
+- Add support for Active Directory an entity analytics provider. {pull}37919[37919]
+- Add AWS AWSHealth metricset. {pull}38370[38370]
+- Add debugging breadcrumb to logs when writing request trace log. {pull}38636[38636]
+- added benchmark input {pull}37437[37437]
+- added benchmark input and discard output {pull}37437[37437]
+- Ensure all responses sent by HTTP Endpoint are HTML-escaped. {pull}39329[39329]
+- Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460]
+- Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455]
 
 *Auditbeat*
 

diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go
@@ -105,13 +105,22 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
 	resp.Body, body, err = copyBody(resp.Body)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err))
+<<<<<<< HEAD
 	} else {
 		respParts = append(respParts,
 			zap.ByteString("http.response.body.content", body),
 			zap.Int("http.response.body.bytes", len(body)),
 			zap.String("http.response.mime_type", resp.Header.Get("Content-Type")),
 		)
+=======
+>>>>>>> 3efb1e8612 (x-pack/filebeat/input/internal/httplog: improve req/resp logging (#39455))
 	}
+	respParts = append(respParts,
+		zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]),
+		zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)),
+		zap.Int("http.response.body.bytes", len(body)),
+		zap.String("http.response.mime_type", resp.Header.Get("Content-Type")),
+	)
 	message, err := httputil.DumpResponse(resp, false)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump response: %s", err))
@@ -172,13 +181,22 @@ func logRequest(log *zap.Logger, req *http.Request, extra ...zapcore.Field) (_ *
 	req.Body, body, err = copyBody(req.Body)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err))
+<<<<<<< HEAD
 	} else {
 		reqParts = append(reqParts,
 			zap.ByteString("http.request.body.content", body),
 			zap.Int("http.request.body.bytes", len(body)),
 			zap.String("http.request.mime_type", req.Header.Get("Content-Type")),
 		)
+=======
+>>>>>>> 3efb1e8612 (x-pack/filebeat/input/internal/httplog: improve req/resp logging (#39455))
 	}
+	reqParts = append(reqParts,
+		zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]),
+		zap.Bool("http.request.body.truncated", maxBodyLen < len(body)),
+		zap.Int("http.request.body.bytes", len(body)),
+		zap.String("http.request.mime_type", req.Header.Get("Content-Type")),
+	)
 	message, err := httputil.DumpRequestOut(req, false)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump request: %s", err))