x-pack/filebeat/input/internal/httplog: improve req/resp logging (#39455

) Attempt to log the request and response bodies and other details even when copying the body has been reported to have failed. (cherry picked from commit 3efb1e8)
elastic · May 9, 2024 · fde0eb7 · fde0eb7
1 parent 8613752
commit fde0eb7
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 14 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -129,6 +129,37 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999]
 - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128]
 - Made GCS input GA and updated docs accordingly. {pull}37127[37127]
+- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160]
+- Prevent CEL input from re-entering the eval loop when an evaluation failed. {pull}37161[37161]
+- Update CEL extensions library to v1.7.0. {pull}37172[37172]
+- Add support for complete URL replacement in HTTPJSON chain steps. {pull}37486[37486]
+- Add support for user-defined query selection in EntraID entity analytics provider. {pull}37653[37653]
+- Update CEL extensions library to v1.8.0 to provide runtime error location reporting. {issue}37304[37304] {pull}37718[37718]
+- Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682]
+- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714]
+- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772]
+- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836]
+- Added experimental version of the Websocket Input. {pull}37774[37774]
+- Add support for PEM-based Okta auth in CEL. {pull}37813[37813]
+- Add Salesforce input. {pull}37331[37331]
+- Add ETW input. {pull}36915[36915]
+- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971]
+- Add logging for cache processor file reads and writes. {pull}38052[38052]
+- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738]
+- Support VPC endpoint for aws-s3 input SQS queue url. {pull}38189[38189]
+- Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237]
+- Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738]
+- Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193]
+- Parse more fields from Elasticsearch slowlogs {pull}38295[38295]
+- Update CEL mito extensions to v1.10.0 to add base64 decode functions. {pull}38504[38504]
+- Add support for Active Directory an entity analytics provider. {pull}37919[37919]
+- Add AWS AWSHealth metricset. {pull}38370[38370]
+- Add debugging breadcrumb to logs when writing request trace log. {pull}38636[38636]
+- added benchmark input {pull}37437[37437]
+- added benchmark input and discard output {pull}37437[37437]
+- Ensure all responses sent by HTTP Endpoint are HTML-escaped. {pull}39329[39329]
+- Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460]
+- Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455]
 
 *Auditbeat*
 

diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go
@@ -109,14 +109,13 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
 	resp.Body, body, err = copyBody(resp.Body)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err))
-	} else {
-		respParts = append(respParts,
-			zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]),
-			zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)),
-			zap.Int("http.response.body.bytes", len(body)),
-			zap.String("http.response.mime_type", resp.Header.Get("Content-Type")),
-		)
 	}
+	respParts = append(respParts,
+		zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]),
+		zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)),
+		zap.Int("http.response.body.bytes", len(body)),
+		zap.String("http.response.mime_type", resp.Header.Get("Content-Type")),
+	)
 	message, err := httputil.DumpResponse(resp, false)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump response: %s", err))
@@ -178,14 +177,13 @@ func logRequest(log *zap.Logger, req *http.Request, maxBodyLen int, extra ...zap
 	req.Body, body, err = copyBody(req.Body)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err))
-	} else {
-		reqParts = append(reqParts,
-			zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]),
-			zap.Bool("http.request.body.truncated", maxBodyLen < len(body)),
-			zap.Int("http.request.body.bytes", len(body)),
-			zap.String("http.request.mime_type", req.Header.Get("Content-Type")),
-		)
 	}
+	reqParts = append(reqParts,
+		zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]),
+		zap.Bool("http.request.body.truncated", maxBodyLen < len(body)),
+		zap.Int("http.request.body.bytes", len(body)),
+		zap.String("http.request.mime_type", req.Header.Get("Content-Type")),
+	)
 	message, err := httputil.DumpRequestOut(req, false)
 	if err != nil {
 		errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump request: %s", err))