Improve logs when there is a timeout error

The agent logs will print the failure message when the connection to
the controller is timeout. Also adding readiness probe to remind
the users of the connectivity issue.

Fixes #822

pkg/agent/apiserver/apiserver.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"path"
 
@@ -28,6 +29,7 @@ import (
 	k8sversion "k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 
 	"github.com/vmware-tanzu/antrea/pkg/agent/apiserver/handlers/addressgroup"
@@ -97,6 +99,14 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier
 	if err != nil {
 		return nil, err
 	}
+	// Add readiness probe to check the status of watchers.
+	check := healthz.NamedCheck("watcher", func(_ *http.Request) error {
+		if npq.GetControllerConnectionStatus() {
+			return nil
+		}
+		return fmt.Errorf("Some watchers may not be connected")
+	})
+	cfg.ReadyzChecks = append(cfg.ReadyzChecks, check)
 	s, err := cfg.New(Name, genericapiserver.NewEmptyDelegate())
 	if err != nil {
 		return nil, err

pkg/agent/controller/networkpolicy/networkpolicy_controller.go

@@ -17,6 +17,7 @@ package networkpolicy
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"sync"
 	"time"
 
@@ -584,6 +585,11 @@ func (w *watcher) watch() {
 		klog.Warningf("Failed to start watch for %s: %v", w.objectType, err)
 		return
 	}
+	// Makesure that watcher is not the type of watch.NewEmptyWatch()
+	if reflect.TypeOf(watcher) == reflect.TypeOf(watch.NewEmptyWatch()) {
+		klog.Warningf("Failed to start watch for %s. Something wrong with the connection?", w.objectType)
+		return
+	}
 
 	klog.Infof("Started watch for %s", w.objectType)
 	w.setConnected(true)