diff --git a/cmd/eksctl/create.go b/cmd/eksctl/create.go index 0c0754d2ba5..b9bc91a3d2c 100644 --- a/cmd/eksctl/create.go +++ b/cmd/eksctl/create.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "time" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -82,7 +81,9 @@ func createClusterCmd() *cobra.Command { fs.StringVar(&kubeconfigPath, "kubeconfig", kubeconfig.DefaultPath, "path to write kubeconfig (incompatible with --auto-kubeconfig)") fs.BoolVar(&setContext, "set-kubeconfig-context", true, "if true then current-context will be set in kubeconfig; if a context is already set then it will be overwritten") - fs.DurationVar(&cfg.AWSOperationTimeout, "aws-api-timeout", 20*time.Minute, "number of seconds after which to timeout AWS API operations") + fs.DurationVar(&cfg.WaitTimeout, "aws-api-timeout", eks.DefaultWaitTimeout, "") + fs.MarkHidden("aws-api-timeout") // TODO deprecate in 0.2.0 + fs.DurationVar(&cfg.WaitTimeout, "timeout", eks.DefaultWaitTimeout, "how long to wait for any of the timely operations") fs.BoolVar(&cfg.Addons.WithIAM.PolicyAmazonEC2ContainerRegistryPowerUser, "full-ecr-access", false, "enable full access to ECR") @@ -162,6 +163,10 @@ func doCreateCluster(cfg *eks.ClusterConfig, name string) error { return err } + if err := cfg.WaitForControlPlane(clientSet); err != nil { + return err + } + // authorise nodes to join if err := cfg.CreateDefaultNodeGroupAuthConfigMap(clientSet); err != nil { return err diff --git a/cmd/eksctl/utils.go b/cmd/eksctl/utils.go index 2d8bf2329d4..70db5db36e1 100644 --- a/cmd/eksctl/utils.go +++ b/cmd/eksctl/utils.go @@ -54,7 +54,8 @@ func waitNodesCmd() *cobra.Command { fs := cmd.Flags() fs.StringVar(&utilsKubeconfigInputPath, "kubeconfig", "kubeconfig", "path to read kubeconfig") - fs.IntVarP(&cfg.MinNodes, "nodes-min", "m", DEFAULT_NODE_COUNT, "minimum nodes to wait for") + fs.IntVarP(&cfg.MinNodes, "nodes-min", "m", DEFAULT_NODE_COUNT, "minimum number of nodes to wait for") + fs.DurationVar(&cfg.WaitTimeout, "timeout", eks.DefaultWaitTimeout, "how long to wait") return cmd } diff --git a/pkg/eks/api.go b/pkg/eks/api.go index 52972055dbf..679a1a7d770 100644 --- a/pkg/eks/api.go +++ b/pkg/eks/api.go @@ -29,6 +29,8 @@ const ( AWSDebugLevel = 5 ) +var DefaultWaitTimeout = 20 * time.Minute + type ClusterProvider struct { // core fields used for config and AWS APIs cfg *ClusterConfig @@ -61,7 +63,7 @@ type ClusterConfig struct { SSHPublicKeyPath string SSHPublicKey []byte - AWSOperationTimeout time.Duration + WaitTimeout time.Duration keyName string clusterRoleARN string diff --git a/pkg/eks/cfn.go b/pkg/eks/cfn.go index 11bbe28f667..189491f71e0 100644 --- a/pkg/eks/cfn.go +++ b/pkg/eks/cfn.go @@ -64,14 +64,14 @@ func (c *ClusterProvider) CreateStack(name string, templateBody []byte, paramete ticker := time.NewTicker(20 * time.Second) defer ticker.Stop() - timer := time.NewTimer(c.cfg.AWSOperationTimeout) + timer := time.NewTimer(c.cfg.WaitTimeout) defer timer.Stop() defer close(errs) for { select { case <-timer.C: - errs <- fmt.Errorf("timed out creating CloudFormation stack %q after %d", name, c.cfg.AWSOperationTimeout) + errs <- fmt.Errorf("timed out creating CloudFormation stack %q after %d", name, c.cfg.WaitTimeout) return case <-ticker.C: diff --git a/pkg/eks/eks.go b/pkg/eks/eks.go index 0b4c1b74a39..7f853a3a9ef 100644 --- a/pkg/eks/eks.go +++ b/pkg/eks/eks.go @@ -7,6 +7,7 @@ import ( "time" "github.com/pkg/errors" + "k8s.io/client-go/kubernetes" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" @@ -72,7 +73,7 @@ func (c *ClusterProvider) createControlPlane(errs chan error) error { ticker := time.NewTicker(20 * time.Second) defer ticker.Stop() - timer := time.NewTimer(c.cfg.AWSOperationTimeout) + timer := time.NewTimer(c.cfg.WaitTimeout) defer timer.Stop() defer close(taskErrs) @@ -81,7 +82,7 @@ func (c *ClusterProvider) createControlPlane(errs chan error) error { for { select { case <-timer.C: - taskErrs <- fmt.Errorf("timed out creating control plane %q after %s", c.cfg.ClusterName, c.cfg.AWSOperationTimeout) + taskErrs <- fmt.Errorf("timed out creating control plane %q after %s", c.cfg.ClusterName, c.cfg.WaitTimeout) return case <-ticker.C: @@ -190,3 +191,28 @@ func (c *ClusterProvider) ListAllTaggedResources() error { // TODO: https://github.com/weaveworks/eksctl/issues/26 return nil } + +func (c *ClusterConfig) WaitForControlPlane(clientSet *kubernetes.Clientset) error { + if _, err := clientSet.ServerVersion(); err == nil { + return nil + } + + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + + timer := time.NewTimer(c.WaitTimeout) + defer timer.Stop() + + for { + select { + case <-ticker.C: + _, err := clientSet.ServerVersion() + if err == nil { + return nil + } + logger.Debug("control plane not ready yet – %s", err.Error()) + case <-timer.C: + return fmt.Errorf("timed out waiting for control plane %q after %s", c.ClusterName, c.WaitTimeout) + } + } +} diff --git a/pkg/eks/nodegroup.go b/pkg/eks/nodegroup.go index 8956062400b..baae93109b6 100644 --- a/pkg/eks/nodegroup.go +++ b/pkg/eks/nodegroup.go @@ -81,8 +81,7 @@ func getNodes(clientSet *clientset.Clientset) (int, error) { } func (c *ClusterConfig) WaitForNodes(clientSet *clientset.Clientset) error { - timeoutAfter := 20 * time.Minute - timer := time.After(timeoutAfter) + timer := time.After(c.WaitTimeout) timeout := false watcher, err := clientSet.Core().Nodes().Watch(metav1.ListOptions{}) if err != nil { @@ -115,7 +114,7 @@ func (c *ClusterConfig) WaitForNodes(clientSet *clientset.Clientset) error { } } if timeout { - return fmt.Errorf("timed out (after %s) waitiing for at least %d nodes to join the cluster and become ready", timeoutAfter, c.MinNodes) + return fmt.Errorf("timed out (after %s) waitiing for at least %d nodes to join the cluster and become ready", c.WaitTimeout, c.MinNodes) } if _, err = getNodes(clientSet); err != nil {