Improve logs when there is a timeout error

The agent logs will print the failure message when the connection to
the controller is timeout. Also adding HealthCheck to remind
the users of the connectivity issue.

Fixes #822

cmd/antrea-agent/agent.go

@@ -17,8 +17,10 @@ package main
 import (
 	"fmt"
 	"net"
+	"net/http"
 	"time"
 
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/informers"
 	"k8s.io/klog"
 
@@ -319,6 +321,13 @@ func run(o *Options) error {
 	if err != nil {
 		return fmt.Errorf("error when creating agent API server: %v", err)
 	}
+	check := healthz.NamedCheck("watcher", func(_ *http.Request) error {
+		if networkPolicyController.GetControllerConnectionStatus() {
+			return nil
+		}
+		return fmt.Errorf("Some watchers may not be connected")
+	})
+	apiServer.GenericAPIServer.AddHealthChecks(check)
 	go apiServer.Run(stopCh)
 
 	// Start PacketIn for features and specify their own reason.

pkg/agent/controller/networkpolicy/networkpolicy_controller.go

@@ -576,6 +576,12 @@ func (w *watcher) watch() {
 		klog.Warningf("Failed to start watch for %s: %v", w.objectType, err)
 		return
 	}
+	// Makesure that watcher is not the type of watch.NewEmptyWatch()
+	_, ok := watcher.(*watch.StreamWatcher)
+	if !ok {
+		klog.Warningf("Failed to start watch for %s. Something wrong with the connection?", w.objectType)
+		return
+	}
 
 	klog.Infof("Started watch for %s", w.objectType)
 	w.setConnected(true)