From c936439160ec165f9094bfff70ce4cb04fc33fda Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sat, 12 Nov 2022 12:09:15 +0000 Subject: [PATCH] wait on startup to get permissions The glbc component needs to get a CRD and have permission to read it, this is not inmidiate when bootstrapping a cluster so, instead of crashing and depending on a external component to restart the pod, we can active poll on this conditions before crashing. This has the benefit of reducing the noise on the logs with panics, that may be misinterpreted by external tools processing the logs. It also improves the bootstrap latency. --- cmd/glbc/main.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/glbc/main.go b/cmd/glbc/main.go index bb731126f6..8d84d68741 100644 --- a/cmd/glbc/main.go +++ b/cmd/glbc/main.go @@ -25,6 +25,7 @@ import ( flag "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/ingress-gce/pkg/frontendconfig" "k8s.io/ingress-gce/pkg/ingparams" "k8s.io/ingress-gce/pkg/instancegroups" @@ -35,6 +36,7 @@ import ( "k8s.io/klog/v2" crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -111,7 +113,20 @@ func main() { // TODO(rramkumar): Reuse this CRD handler for other CRD's coming. crdHandler := crd.NewCRDHandler(crdClient) backendConfigCRDMeta := backendconfig.CRDMeta() - if _, err := crdHandler.EnsureCRD(backendConfigCRDMeta, true); err != nil { + // The CRD and/or its permission may take a while, wait for 1 minute to avoid + // crashing so we can reduce the noise on the logs. + if err := wait.PollImmediate(3*time.Second, 1*time.Minute, func() (bool, error) { + _, err := crdHandler.EnsureCRD(backendConfigCRDMeta, true) + // retry if kube-apiserver returns a not found or forbidden error + if apierrors.IsNotFound(err) || apierrors.IsForbidden(err) { + return false, nil + } + // crash on any other error + if err != nil { + return false, err + } + return true, nil + }); err != nil { klog.Fatalf("Failed to ensure BackendConfig CRD: %v", err) }