x-pack/filebeat/input/cel: suppress and log max http request retry er…

…rors

The retryablehttp.Client will return an error based on the ErrorHandler
field when the number of requests exceeds the maximum configuration.
In the case that the field is nil, this is a non-nil error that causes
issues with some APIs and does not allow the CEL code to evaluate the
HTTP response status. So add a retryablehttp.ErrorHandler that will log
the failure, but return the response unaltered and a nil error.

CHANGELOG.next.asciidoc

@@ -213,6 +213,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128]
 - Add request trace logging to http_endpoint input. {issue}36951[36951] {pull}36957[36957]
 - Made GCS input GA and updated docs accordingly. {pull}37127[37127]
+- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160]
 
 *Auditbeat*
 

x-pack/filebeat/input/cel/input.go

@@ -730,14 +730,16 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client,
 	c.CheckRedirect = checkRedirect(cfg.Resource, log)
 
 	if cfg.Resource.Retry.getMaxAttempts() > 1 {
+		maxAttempts := cfg.Resource.Retry.getMaxAttempts()
 		c = (&retryablehttp.Client{
 			HTTPClient:   c,
 			Logger:       newRetryLog(log),
 			RetryWaitMin: cfg.Resource.Retry.getWaitMin(),
 			RetryWaitMax: cfg.Resource.Retry.getWaitMax(),
-			RetryMax:     cfg.Resource.Retry.getMaxAttempts(),
+			RetryMax:     maxAttempts,
 			CheckRetry:   retryablehttp.DefaultRetryPolicy,
 			Backoff:      retryablehttp.DefaultBackoff,
+			ErrorHandler: retryErrorHandler(maxAttempts, log),
 		}).StandardClient()
 	}
 
@@ -831,6 +833,16 @@ func checkRedirect(cfg *ResourceConfig, log *logp.Logger) func(*http.Request, []
 	}
 }
 
+// retryErrorHandler returns a retryablehttp.ErrorHandler that will log retry resignation
+// but return the last retry attempt's response and a nil error so that the CEL code
+// can evaluate the response status itself.
+func retryErrorHandler(max int, log *logp.Logger) retryablehttp.ErrorHandler {
+	return func(resp *http.Response, err error, numTries int) (*http.Response, error) {
+		log.Warnw("giving up retries", "method", resp.Request.Method, "url", resp.Request.URL, "retries", max+1)
+		return resp, nil
+	}
+}
+
 func newRateLimiterFromConfig(cfg *ResourceConfig) *rate.Limiter {
 	r := rate.Inf
 	b := 1