Handling better DeadlineExceeded errors from Docker API (elastic#15336)

jsoriano · Jan 8, 2020 · 7c562d9 · 7c562d9
1 parent 7aee956
commit 7c562d9
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -329,6 +329,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Add a friendly log message when a request to docker has exceeded the deadline. {pull}15336[15336]
 - Decouple Debug logging from fail_on_error logic for rename, copy, truncate processors {pull}12451[12451]
 - Add an option to append to existing logs rather than always rotate on start. {pull}11953[11953]
 - Add `network` condition to processors for matching IP addresses against CIDRs. {pull}10743[10743]

diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go
@@ -308,13 +308,18 @@ func (w *watcher) watch() {
 
 			case err := <-errors:
 				// Restart watch call
-				logp.Err("Error watching for docker events: %v", err)
+				if err == context.DeadlineExceeded {
+					logp.Info("Context deadline exceeded for docker request, restarting watch call")
+				} else {
+					logp.Err("Error watching for docker events: %v", err)
+				}
+
 				time.Sleep(1 * time.Second)
 				break WATCH
 
 			case <-tickChan.C:
 				if time.Since(w.lastWatchReceivedEventTime) > dockerEventsWatchPityTimerTimeout {
-					logp.Info("No events received withing %s, restarting watch call", dockerEventsWatchPityTimerTimeout)
+					logp.Info("No events received within %s, restarting watch call", dockerEventsWatchPityTimerTimeout)
 					time.Sleep(1 * time.Second)
 					break WATCH
 				}