Fix terminating pod autodiscover issue (elastic#20084)

CHANGELOG.next.asciidoc

@@ -148,6 +148,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix config reload metrics (`libbeat.config.module.start/stops/running`). {pull}19168[19168]
 - Fix metrics hints builder to avoid wrong container metadata usage when port is not exposed {pull}18979[18979]
 - Server-side TLS config now validates certificate and key are both specified {pull}19584[19584]
+- Fix terminating pod autodiscover issue. {pull}20084[20084]
 - Fix seccomp policy for calls to `chmod` and `chown`. {pull}20054[20054]
 - Output errors when Kibana index pattern setup fails. {pull}20121[20121]
 

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -138,14 +138,30 @@ func (p *pod) OnUpdate(obj interface{}) {
 	switch pod.Status.Phase {
 	case kubernetes.PodSucceeded, kubernetes.PodFailed:
 		// If Pod is in a phase where all containers in the have terminated emit a stop event
-		p.logger.Debugf("Watcher Pod update (terminating): %+v", obj)
+		p.logger.Debugf("Watcher Pod update (terminated): %+v", obj)
 		time.AfterFunc(p.config.CleanupTimeout, func() { p.emit(pod, "stop") })
 		return
 	case kubernetes.PodPending:
 		p.logger.Debugf("Watcher Pod update (pending): don't know what to do with this Pod yet, skipping for now: %+v", obj)
 		return
 	}
 
+	// here handle the case when a Pod is in `Terminating` phase.
+	// In this case the pod is neither `PodSucceeded` nor `PodFailed` and
+	// hence requires special handling.
+	if pod.GetObjectMeta().GetDeletionTimestamp() != nil {
+		p.logger.Debugf("Watcher Pod update (terminating): %+v", obj)
+		// Pod is terminating, don't reload its configuration and ignore the event
+		// if some pod is still running, we will receive more events when containers
+		// terminate.
+		for _, container := range pod.Status.ContainerStatuses {
+			if container.State.Running != nil {
+				return
+			}
+		}
+		time.AfterFunc(p.config.CleanupTimeout, func() { p.emit(pod, "stop") })
+	}
+
 	p.logger.Debugf("Watcher Pod update: %+v", obj)
 	p.emit(pod, "stop")
 	p.emit(pod, "start")