diff --git a/.docs/commands/havener.md b/.docs/commands/havener.md index 644a2a30..1538ff62 100644 --- a/.docs/commands/havener.md +++ b/.docs/commands/havener.md @@ -17,7 +17,7 @@ See the individual commands to get the complete overview. ### Options ``` - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --fatal fatal output - level 1 diff --git a/.docs/commands/havener_events.md b/.docs/commands/havener_events.md index 34a4596f..7a7e8dbe 100644 --- a/.docs/commands/havener_events.md +++ b/.docs/commands/havener_events.md @@ -23,7 +23,7 @@ havener events [flags] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/.docs/commands/havener_logs.md b/.docs/commands/havener_logs.md index 88861f9b..e1ef9dbd 100644 --- a/.docs/commands/havener_logs.md +++ b/.docs/commands/havener_logs.md @@ -31,7 +31,7 @@ havener logs [flags] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/.docs/commands/havener_node-exec.md b/.docs/commands/havener_node-exec.md index fc72c571..5a31a2b2 100644 --- a/.docs/commands/havener_node-exec.md +++ b/.docs/commands/havener_node-exec.md @@ -4,23 +4,24 @@ Execute command on Kubernetes node ### Synopsis -Execute a command on a node. +Execute a command on a node -This executes a command directly on the node itself. Therefore, havener creates -a temporary pod which enables the user to access the shell of the node. The pod +Execute a command directly on the node itself. For this, havener creates a +temporary pod, which enables the user to access the shell of the node. The pod is deleted automatically afterwards. The command can be omitted which will result in the default command: /bin/sh. For -example 'havener node-exec foo' will search for a node named 'foo' and open a -shell if found. +example havener node-exec foo will search for a node named 'foo' and open a +shell if the node can be found. -Typically, the TTY flag does have to be specified. By definition, if one one -target node is provided, it is assumed that TTY is desired and STDIN is attached -to the remote process. Analog, for the distributed mode with multiple nodes, -no TTY is set, and the STDIN is multiplexed into each remote process. +When more than one node is specified, it will execute the command on all nodes. +In this distributed mode, both passing the StdIn as well as TTY mode are not +available. By default, the number of parallel node executions is limited to 5 +in parallel in order to not create to many requests at the same time. This +value can be overwritten. Handle with care. -If you run the 'node-exec' without any additional arguments, it will print a -list of available nodes. +If you run the node-exec without any additional arguments, it will print a +list of available nodes in the cluster. For convenience, if the target node name all is used, havener will look up all nodes automatically. @@ -34,12 +35,13 @@ havener node-exec [flags] [[,,...]] [] ### Options ``` - --block show distributed shell output as block for each node + -i, --stdin Pass stdin to the container + -t, --tty Stdin is a TTY + --image string Container image used for helper pod (from which the root-shell is accessed) (default "docker.io/library/alpine") + --timeout duration Timeout for the setup of the helper pod (default 30s) + --max-parallel int Number of parallel executions (value less or equal than zero means unlimited) (default 5) + --block Show distributed shell output as block for each node -h, --help help for node-exec - --image string set image for helper pod from which the root-shell is accessed (default "alpine") - --max-parallel int number of parallel executions (defaults to number of nodes) - --no-tty do not allocate pseudo-terminal for command execution - --timeout int set timout in seconds for the setup of the helper pod (default 10) ``` ### Options inherited from parent commands @@ -48,7 +50,7 @@ havener node-exec [flags] [[,,...]] [] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/.docs/commands/havener_pod-exec.md b/.docs/commands/havener_pod-exec.md index 9cae4190..c763463b 100644 --- a/.docs/commands/havener_pod-exec.md +++ b/.docs/commands/havener_pod-exec.md @@ -4,26 +4,26 @@ Execute command on Kubernetes pod ### Synopsis -Execute a shell command on a pod. +Execute a command on a pod -This is similar to the kubectl exec command with just a slightly -different syntax. In contrast to kubectl, you do not have to specify -the namespace of the pod. +This is similar to the kubectl exec command with just a slightly different +syntax. In contrast to kubectl, you do not have to specify the namespace +of the pod. -If no namespace is given, havener will search all namespaces for -a pod that matches the name. +If no namespace is given, havener will search all namespaces for a pod that +matches the name. -Also, you can omit the command which will result in the default -command: /bin/sh. For example 'havener pod-exec api-0' will search -for a pod named 'api-0' in all namespaces and open a shell if found. +Also, you can omit the command which will result in the default command: /bin/sh. +For example havener pod-exec api-0 will search for a pod named api-0 in all +namespaces and open a shell if found. -In case no container name is given, havener will assume you want to -execute the command in the first container found in the pod. +In case no container name is given, havener will assume you want to execute the +command in the first container found in the pod. If you run the 'pod-exec' without any additional arguments, it will print a list of available pods. -For convenience, if the target pod name _all_ is used, havener will look up +For convenience, if the target pod name all is used, havener will look up all pods in all namespaces automatically. @@ -34,9 +34,10 @@ havener pod-exec [flags] [[/][/container]] [] ### Options ``` - --block show distributed shell output as block for each pod - -h, --help help for pod-exec - --no-tty do not allocate pseudo-terminal for command execution + -i, --stdin Pass stdin to the container + -t, --tty Stdin is a TTY + --block show distributed shell output as block for each pod + -h, --help help for pod-exec ``` ### Options inherited from parent commands @@ -45,7 +46,7 @@ havener pod-exec [flags] [[/][/container]] [] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/.docs/commands/havener_top.md b/.docs/commands/havener_top.md index d1ea7d5c..5924cab1 100644 --- a/.docs/commands/havener_top.md +++ b/.docs/commands/havener_top.md @@ -33,7 +33,7 @@ havener top [flags] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/.docs/commands/havener_version.md b/.docs/commands/havener_version.md index 1deca06b..b743102a 100644 --- a/.docs/commands/havener_version.md +++ b/.docs/commands/havener_version.md @@ -22,7 +22,7 @@ havener version [flags] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/.docs/commands/havener_watch.md b/.docs/commands/havener_watch.md index bed8a78c..32cd0eb3 100644 --- a/.docs/commands/havener_watch.md +++ b/.docs/commands/havener_watch.md @@ -26,7 +26,7 @@ havener watch [flags] --debug debug output - level 5 --error error output - level 2 --fatal fatal output - level 1 - --kubeconfig string Kubernetes configuration file (default "~/.kube/config") + --kubeconfig string Kubernetes configuration (default "~/.kube/config") --terminal-height int disable autodetection and specify an explicit terminal height (default -1) --terminal-width int disable autodetection and specify an explicit terminal width (default -1) --trace trace output - level 6 diff --git a/go.mod b/go.mod index bc9854a2..3800970a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/gonvenience/wait v1.0.3 github.com/gonvenience/wrap v1.2.0 github.com/lucasb-eyer/go-colorful v1.2.0 + github.com/mattn/go-isatty v0.0.18 github.com/onsi/ginkgo/v2 v2.17.3 github.com/onsi/gomega v1.33.1 github.com/spf13/cobra v1.8.0 @@ -58,7 +59,6 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect - github.com/mattn/go-isatty v0.0.18 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/spdystream v0.2.0 // indirect diff --git a/internal/cmd/common.go b/internal/cmd/common.go index 9bd0d42e..51ec7eea 100644 --- a/internal/cmd/common.go +++ b/internal/cmd/common.go @@ -21,41 +21,14 @@ package cmd import ( - "io" + "os" "github.com/gonvenience/wrap" + "github.com/mattn/go-isatty" ) -// duplicateReader creates a given number of io.Reader duplicates based on the -// provided input reader. This way it is possible to use one input reader for -// more than one consumer. -func duplicateReader(reader io.Reader, count int) []io.Reader { - writers := []io.Writer{} - readers := []io.Reader{} - for i := 0; i < count; i++ { - r, w := io.Pipe() - writers = append(writers, w) - readers = append(readers, r) - } - - writer := io.MultiWriter(writers...) - go func() { - if _, err := io.Copy(writer, reader); err != nil { - panic(err) - } - - for i := range writers { - if w, ok := writers[i].(io.Closer); ok { - w.Close() - } - } - }() - - return readers -} - func combineErrorsFromChannel(context string, c chan error) error { - errors := []error{} + var errors []error for err := range c { if err != nil { errors = append(errors, err) @@ -70,3 +43,7 @@ func combineErrorsFromChannel(context string, c chan error) error { return wrap.Errors(errors, context) } } + +func isTerminal(fd uintptr) bool { return isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd) } + +func isStdinTerminal() bool { return isTerminal(os.Stdin.Fd()) } diff --git a/internal/cmd/events.go b/internal/cmd/events.go index b7bcd87d..6f75b6d1 100644 --- a/internal/cmd/events.go +++ b/internal/cmd/events.go @@ -21,7 +21,6 @@ package cmd import ( - "context" "fmt" "strings" "time" @@ -86,7 +85,7 @@ func retrieveClusterEvents(hvnr havener.Havener) error { continue } - watcher, err := hvnr.Client().CoreV1().Events(namespace).Watch(context.TODO(), metav1.ListOptions{}) + watcher, err := hvnr.Client().CoreV1().Events(namespace).Watch(hvnr.Context(), metav1.ListOptions{}) if err != nil { return fmt.Errorf("failed to setup event watcher: %w", err) } diff --git a/internal/cmd/nexec.go b/internal/cmd/nexec.go index 1f1e7c17..a75fae06 100644 --- a/internal/cmd/nexec.go +++ b/internal/cmd/nexec.go @@ -24,60 +24,79 @@ import ( "fmt" "io" "os" + "os/user" + "runtime" "strings" "sync" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/gonvenience/bunt" - "github.com/gonvenience/term" "github.com/homeport/havener/pkg/havener" "github.com/spf13/cobra" ) -const nodeDefaultCommand = "/bin/sh" - -var ( - nodeExecNoTty bool - nodeExecImage string - nodeExecTimeout int - nodeExecBlock bool - defaultImage = "alpine" - defaultTimeout = 10 - nodeExecMaxParallel = 10 +const ( + nodeExecDefaultImage = "docker.io/library/alpine" + nodeExecDefaultTimeout = 30 * time.Second + nodeExecDefaultMaxParallel = 5 + nodeExecDefaultCommand = "/bin/sh" ) +var nodeExecCmdSettings struct { + stdin bool + tty bool + notty bool + image string + maxParallel int + timeout time.Duration + printAsBlock bool +} + // nodeExecCmd represents the node-exec command var nodeExecCmd = &cobra.Command{ Use: "node-exec [flags] [[,,...]] []", Aliases: []string{"ne"}, Short: "Execute command on Kubernetes node", - Long: bunt.Sprintf(`Execute a command on a node. + Long: bunt.Sprintf(`*Execute a command on a node* -This executes a command directly on the node itself. Therefore, havener creates -a temporary pod which enables the user to access the shell of the node. The pod +Execute a command directly on the node itself. For this, *havener* creates a +temporary pod, which enables the user to access the shell of the node. The pod is deleted automatically afterwards. The command can be omitted which will result in the default command: _%s_. For -example 'havener node-exec foo' will search for a node named 'foo' and open a -shell if found. +example _havener node-exec foo_ will search for a node named 'foo' and open a +shell if the node can be found. -Typically, the TTY flag does have to be specified. By definition, if one one -target node is provided, it is assumed that TTY is desired and STDIN is attached -to the remote process. Analog, for the distributed mode with multiple nodes, -no TTY is set, and the STDIN is multiplexed into each remote process. +When more than one node is specified, it will execute the command on all nodes. +In this distributed mode, both passing the StdIn as well as TTY mode are not +available. By default, the number of parallel node executions is limited to %d +in parallel in order to not create to many requests at the same time. This +value can be overwritten. Handle with care. -If you run the 'node-exec' without any additional arguments, it will print a -list of available nodes. +If you run the _node-exec_ without any additional arguments, it will print a +list of available nodes in the cluster. -For convenience, if the target node name _all_ is used, havener will look up +For convenience, if the target node name _all_ is used, *havener* will look up all nodes automatically. -`, nodeDefaultCommand), +`, nodeExecDefaultCommand, nodeExecDefaultMaxParallel), SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { + // Check edge case for deprecated command-line flag + if cmd.Flags().Changed("no-tty") { + // Bail out if both the new and the old flag are used at the same time + if cmd.Flags().Changed("tty") { + return fmt.Errorf("cannot use --no-tty and --tty at the same time") + } + + // If only --no-tty is used, continue to accept its input + nodeExecCmdSettings.tty = !nodeExecCmdSettings.notty + } + hvnr, err := havener.NewHavener(havener.WithContext(cmd.Context()), havener.WithKubeConfigPath(kubeConfig)) if err != nil { return fmt.Errorf("unable to get access to cluster: %w", err) @@ -90,11 +109,17 @@ all nodes automatically. func init() { rootCmd.AddCommand(nodeExecCmd) - nodeExecCmd.PersistentFlags().BoolVar(&nodeExecNoTty, "no-tty", false, "do not allocate pseudo-terminal for command execution") - nodeExecCmd.PersistentFlags().StringVar(&nodeExecImage, "image", defaultImage, "set image for helper pod from which the root-shell is accessed") - nodeExecCmd.PersistentFlags().IntVar(&nodeExecTimeout, "timeout", defaultTimeout, "set timout in seconds for the setup of the helper pod") - nodeExecCmd.PersistentFlags().BoolVar(&nodeExecBlock, "block", false, "show distributed shell output as block for each node") - nodeExecCmd.PersistentFlags().IntVar(&nodeExecMaxParallel, "max-parallel", 0, "number of parallel executions (defaults to number of nodes)") + nodeExecCmd.Flags().SortFlags = false + nodeExecCmd.Flags().BoolVarP(&nodeExecCmdSettings.stdin, "stdin", "i", false, "Pass stdin to the container") + nodeExecCmd.Flags().BoolVarP(&nodeExecCmdSettings.tty, "tty", "t", false, "Stdin is a TTY") + nodeExecCmd.Flags().StringVar(&nodeExecCmdSettings.image, "image", nodeExecDefaultImage, "Container image used for helper pod (from which the root-shell is accessed)") + nodeExecCmd.Flags().DurationVar(&nodeExecCmdSettings.timeout, "timeout", nodeExecDefaultTimeout, "Timeout for the setup of the helper pod") + nodeExecCmd.Flags().IntVar(&nodeExecCmdSettings.maxParallel, "max-parallel", nodeExecDefaultMaxParallel, "Number of parallel executions (value less or equal than zero means unlimited)") + nodeExecCmd.Flags().BoolVar(&nodeExecCmdSettings.printAsBlock, "block", false, "Show distributed shell output as block for each node") + + // Deprecated/old flags + nodeExecCmd.Flags().BoolVar(&nodeExecCmdSettings.notty, "no-tty", false, "do not allocate pseudo-terminal for command execution") + _ = nodeExecCmd.Flags().MarkDeprecated("no-tty", "use --tty flag instead") } func execInClusterNodes(hvnr havener.Havener, args []string) error { @@ -114,7 +139,7 @@ func execInClusterNodes(hvnr havener.Havener, args []string) error { } case len(args) == 1: // only node name is given - input, command = args[0], []string{nodeDefaultCommand} + input, command = args[0], []string{nodeExecDefaultCommand} nodes, err = lookupNodesByName(hvnr, input) if err != nil { return err @@ -124,43 +149,55 @@ func execInClusterNodes(hvnr havener.Havener, args []string) error { return availableNodesError(hvnr, "no node name and command specified") } - // In case the current process does not run in a terminal, disable the - // default TTY behavior. - if !term.IsTerminal() { - nodeExecNoTty = true + if !isStdinTerminal() { + nodeExecCmdSettings.tty = false + } + + var in io.Reader + if nodeExecCmdSettings.stdin { + in = os.Stdin + } + + var nodeExecHelperPodConfig = havener.NodeExecHelperPodConfig{ + Annotations: map[string]string{}, + ContainerImage: nodeExecCmdSettings.image, + ContainerCmd: []string{"/bin/sleep", "8h"}, + WaitTimeout: nodeExecCmdSettings.timeout, } + nodeExecHelperPodConfig.Annotations["originator"] = originator() + // Single node mode, use default streams and run node execute function if len(nodes) == 1 { return hvnr.NodeExec( nodes[0], - nodeExecImage, - nodeExecTimeout, - command, - os.Stdin, - os.Stdout, - os.Stderr, - !nodeExecNoTty, + nodeExecHelperPodConfig, + havener.ExecConfig{ + Command: command, + Stdin: in, + Stdout: os.Stdout, + Stderr: os.Stderr, + TTY: nodeExecCmdSettings.tty, + }, ) } - // In distributed shell mode, TTY is forced to be disabled - nodeExecNoTty = true + // In distributed shell mode, stdin is not piped through and TTY is forced to be disabled + nodeExecCmdSettings.stdin = false + nodeExecCmdSettings.tty = false - // In case nothing is configured, all nodes will be executed concurrently - if nodeExecMaxParallel <= 0 || nodeExecMaxParallel > len(nodes) { - nodeExecMaxParallel = len(nodes) + // In case the user wants everything done in parallel, increase the max value + if nodeExecCmdSettings.maxParallel <= 0 { + nodeExecCmdSettings.maxParallel = len(nodes) } type task struct { - node corev1.Node - reader io.Reader + node corev1.Node } var ( wg = &sync.WaitGroup{} tasks = make(chan task, len(nodes)) - readers = duplicateReader(os.Stdin, len(nodes)) output = make(chan OutputMsg) errors = make(chan error, len(nodes)) printer = make(chan bool, 1) @@ -168,25 +205,26 @@ func execInClusterNodes(hvnr havener.Havener, args []string) error { // Fill task queue with the list of nodes to be processed for i := range nodes { - tasks <- task{reader: readers[i], node: nodes[i]} + tasks <- task{node: nodes[i]} } close(tasks) // Start n task workers to work on task queue - wg.Add(nodeExecMaxParallel) - for i := 0; i < nodeExecMaxParallel; i++ { + wg.Add(nodeExecCmdSettings.maxParallel) + for i := 0; i < nodeExecCmdSettings.maxParallel; i++ { go func() { defer wg.Done() for task := range tasks { errors <- hvnr.NodeExec( task.node, - nodeExecImage, - nodeExecTimeout, - command, - task.reader, - chanWriter("StdOut", task.node.Name, output), - chanWriter("StdErr", task.node.Name, output), - !nodeExecNoTty, + nodeExecHelperPodConfig, + havener.ExecConfig{ + Command: command, + Stdin: nil, // Disabled for now until reliable input duplication works + Stdout: chanWriter("StdOut", task.node.Name, output), + Stderr: chanWriter("StdErr", task.node.Name, output), + TTY: nodeExecCmdSettings.tty, + }, ) } }() @@ -194,7 +232,7 @@ func execInClusterNodes(hvnr havener.Havener, args []string) error { // Start the respective output printer in a separate Go routine go func() { - if nodeExecBlock { + if nodeExecCmdSettings.printAsBlock { PrintOutputMessageAsBlock(output) } else { @@ -212,6 +250,28 @@ func execInClusterNodes(hvnr havener.Havener, args []string) error { return combineErrorsFromChannel("node command execution failed", errors) } +func originator() string { + if version == "" { + version = "development version" + } + + var username = "unknown" + if user, err := user.Current(); err == nil { + username = user.Name + } + + var hostname = "unknown" + if name, err := os.Hostname(); err == nil { + hostname = name + } + + return fmt.Sprintf("havener %s (%s/%s), %s@%s", + version, + runtime.GOOS, runtime.GOARCH, + username, hostname, + ) +} + func lookupNodesByName(h havener.Havener, input string) ([]corev1.Node, error) { if input == "all" { return h.ListNodes() diff --git a/internal/cmd/pexec.go b/internal/cmd/pexec.go index e35656a3..bcf4e3f7 100644 --- a/internal/cmd/pexec.go +++ b/internal/cmd/pexec.go @@ -30,13 +30,15 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/gonvenience/term" + "github.com/gonvenience/bunt" "github.com/homeport/havener/pkg/havener" "github.com/spf13/cobra" "github.com/spf13/viper" ) -const podDefaultCommand = "/bin/sh" +const ( + podExecDefaultCommand = "/bin/sh" +) type target struct { namespace string @@ -44,41 +46,54 @@ type target struct { containerName string } -var ( - podExecNoTty bool - podExecBlock bool -) +var podExecCmdSettings struct { + stdin bool + tty bool + notty bool + printAsBlock bool +} // podExecCmd represents the pod-exec command var podExecCmd = &cobra.Command{ Use: "pod-exec [flags] [[/][/container]] []", Aliases: []string{"pe"}, Short: "Execute command on Kubernetes pod", - Long: `Execute a shell command on a pod. + Long: bunt.Sprintf(`*Execute a command on a pod* -This is similar to the kubectl exec command with just a slightly -different syntax. In contrast to kubectl, you do not have to specify -the namespace of the pod. +This is similar to the kubectl exec command with just a slightly different +syntax. In contrast to kubectl, you do not have to specify the namespace +of the pod. -If no namespace is given, havener will search all namespaces for -a pod that matches the name. +If no namespace is given, *havener* will search all namespaces for a pod that +matches the name. -Also, you can omit the command which will result in the default -command: ` + podDefaultCommand + `. For example 'havener pod-exec api-0' will search -for a pod named 'api-0' in all namespaces and open a shell if found. +Also, you can omit the command which will result in the default command: %s. +For example _havener pod-exec api-0_ will search for a pod named _api-0_ in all +namespaces and open a shell if found. -In case no container name is given, havener will assume you want to -execute the command in the first container found in the pod. +In case no container name is given, *havener* will assume you want to execute the +command in the first container found in the pod. If you run the 'pod-exec' without any additional arguments, it will print a list of available pods. -For convenience, if the target pod name _all_ is used, havener will look up +For convenience, if the target pod name _all_ is used, *havener* will look up all pods in all namespaces automatically. -`, +`, podExecDefaultCommand), SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { + // Check edge case for deprecated command-line flag + if cmd.Flags().Changed("no-tty") { + // Bail out if both the new and the old flag are used at the same time + if cmd.Flags().Changed("tty") { + return fmt.Errorf("cannot use --no-tty and --tty at the same time") + } + + // If only --no-tty is used, continue to accept its input + podExecCmdSettings.tty = !podExecCmdSettings.notty + } + hvnr, err := havener.NewHavener(havener.WithContext(cmd.Context()), havener.WithKubeConfigPath(kubeConfig)) if err != nil { return fmt.Errorf("unable to get access to cluster: %w", err) @@ -91,8 +106,14 @@ all pods in all namespaces automatically. func init() { rootCmd.AddCommand(podExecCmd) - podExecCmd.Flags().BoolVar(&podExecNoTty, "no-tty", false, "do not allocate pseudo-terminal for command execution") - podExecCmd.Flags().BoolVar(&podExecBlock, "block", false, "show distributed shell output as block for each pod") + podExecCmd.Flags().SortFlags = false + podExecCmd.Flags().BoolVarP(&podExecCmdSettings.stdin, "stdin", "i", false, "Pass stdin to the container") + podExecCmd.Flags().BoolVarP(&podExecCmdSettings.tty, "tty", "t", false, "Stdin is a TTY") + podExecCmd.Flags().BoolVar(&podExecCmdSettings.printAsBlock, "block", false, "show distributed shell output as block for each pod") + + // Deprecated/old flags + podExecCmd.Flags().BoolVar(&podExecCmdSettings.notty, "no-tty", false, "do not allocate pseudo-terminal for command execution") + _ = podExecCmd.Flags().MarkDeprecated("no-tty", "use --tty flag instead") } func execInClusterPods(hvnr havener.Havener, args []string) error { @@ -113,7 +134,7 @@ func execInClusterPods(hvnr havener.Havener, args []string) error { } case len(args) == 1: // only pod is given - input, command = args[0], []string{podDefaultCommand} + input, command = args[0], []string{podExecDefaultCommand} podMap, err = lookupPodsByName(hvnr, input) if err != nil { return err @@ -131,10 +152,13 @@ func execInClusterPods(hvnr havener.Havener, args []string) error { } } - // In case the current process does not run in a terminal, disable the - // default TTY behavior. - if !term.IsTerminal() { - podExecNoTty = true + if !isStdinTerminal() { + podExecCmdSettings.tty = false + } + + var in io.Reader + if podExecCmdSettings.stdin { + in = os.Stdin } // Single pod mode, use default streams and run pod execute function @@ -142,25 +166,25 @@ func execInClusterPods(hvnr havener.Havener, args []string) error { for pod, containers := range podMap { for i := range containers { return hvnr.PodExec( - pod, - containers[i], - command, - os.Stdin, - os.Stdout, - os.Stderr, - !podExecNoTty, + pod, containers[i], + havener.ExecConfig{ + Command: command, + Stdin: in, + Stdout: os.Stdout, + Stderr: os.Stderr, + TTY: podExecCmdSettings.tty, + }, ) } - } } // In distributed shell mode, TTY is forced to be disabled - podExecNoTty = true + podExecCmdSettings.stdin = false + podExecCmdSettings.tty = false var ( wg = &sync.WaitGroup{} - readers = duplicateReader(os.Stdin, countContainers) output = make(chan OutputMsg) errors = make(chan error, countContainers) printer = make(chan bool, 1) @@ -170,26 +194,27 @@ func execInClusterPods(hvnr havener.Havener, args []string) error { for pod, containers := range podMap { for i := range containers { wg.Add(1) - go func(pod *corev1.Pod, container string, reader io.Reader) { + go func(pod *corev1.Pod, container string) { defer wg.Done() origin := fmt.Sprintf("%s/%s", pod.Name, container) errors <- hvnr.PodExec( - pod, - container, - command, - reader, - chanWriter("StdOut", origin, output), - chanWriter("StdErr", origin, output), - !podExecNoTty, + pod, container, + havener.ExecConfig{ + Command: command, + Stdin: nil, // Disabled for now until reliable input duplication works + Stdout: chanWriter("StdOut", origin, output), + Stderr: chanWriter("StdErr", origin, output), + TTY: podExecCmdSettings.tty, + }, ) - }(pod, containers[i], readers[counter]) + }(pod, containers[i]) counter++ } } // Start the respective output printer in a separate Go routine go func() { - if podExecBlock { + if podExecCmdSettings.printAsBlock { PrintOutputMessageAsBlock(output) } else { diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 5376ef8a..cb8a6e33 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -98,7 +98,7 @@ func Execute() { } func init() { - kubeConfigDefault, err := havener.KubeConfigDefault() + kubeConfigDefault, err := havener.KubeConfig() if err != nil { panic(err) } @@ -106,7 +106,7 @@ func init() { rootCmd.Flags().SortFlags = false rootCmd.PersistentFlags().SortFlags = false - rootCmd.PersistentFlags().StringVar(&kubeConfig, "kubeconfig", kubeConfigDefault, "Kubernetes configuration file") + rootCmd.PersistentFlags().StringVar(&kubeConfig, "kubeconfig", kubeConfigDefault, "Kubernetes configuration") rootCmd.PersistentFlags().Int("terminal-width", -1, "disable autodetection and specify an explicit terminal width") rootCmd.PersistentFlags().Int("terminal-height", -1, "disable autodetection and specify an explicit terminal height") diff --git a/pkg/havener/common.go b/pkg/havener/common.go index b590aca6..baf1e681 100644 --- a/pkg/havener/common.go +++ b/pkg/havener/common.go @@ -41,9 +41,16 @@ import ( "gopkg.in/yaml.v3" ) -// KubeConfigDefault returns assumed default locaation of the Kubernetes -// configuration, which is expected to be `$HOME/.kube/config`. -func KubeConfigDefault() (string, error) { +// KubeConfig returns the path to the Kubernetes configuration, +// this is either whatever is set in KUBECONFIG, +// or the well known default location `$HOME/.kube/config` +func KubeConfig() (string, error) { + // In case `KUBECONFIG` environment variable is set, this will take precedence + if value, ok := os.LookupEnv("KUBECONFIG"); ok { + return value, nil + } + + // Otherwise, default to the well known location home, err := os.UserHomeDir() if err != nil { return "", fmt.Errorf("unable to get home directory: %w", err) diff --git a/pkg/havener/havener.go b/pkg/havener/havener.go index ce164aac..8c3c5cfd 100644 --- a/pkg/havener/havener.go +++ b/pkg/havener/havener.go @@ -32,7 +32,6 @@ package havener import ( "context" "fmt" - "io" "os" "strconv" "time" @@ -109,8 +108,8 @@ type Havener interface { TopDetails() (*TopDetails, error) RetrieveLogs(parallelDownloads int, target string, includeConfigFiles bool) error - PodExec(pod *corev1.Pod, container string, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, tty bool) error - NodeExec(node corev1.Node, containerImage string, timeoutSeconds int, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, tty bool) error + PodExec(pod *corev1.Pod, container string, execConfig ExecConfig) error + NodeExec(node corev1.Node, hlpPodConfig NodeExecHelperPodConfig, execConfig ExecConfig) error } // Option provides a way to set specific settings for creating the Havener setup @@ -137,15 +136,9 @@ func NewHavener(opts ...Option) (hvnr *Hvnr, err error) { hvnr.ctx = context.Background() } - // In case `KUBECONFIG` environment variable is set, this will take - // precedence over command line flag or default value - if value, ok := os.LookupEnv("KUBECONFIG"); ok { - hvnr.kubeConfigPath = value - } - // In case there is no Kubernetes configuration set, use the default if hvnr.kubeConfigPath == "" { - hvnr.kubeConfigPath, err = KubeConfigDefault() + hvnr.kubeConfigPath, err = KubeConfig() if err != nil { return nil, fmt.Errorf("failed to look-up default kube config: %w", err) } diff --git a/pkg/havener/kubexec.go b/pkg/havener/kubexec.go index 02eb2183..93def40d 100644 --- a/pkg/havener/kubexec.go +++ b/pkg/havener/kubexec.go @@ -21,27 +21,46 @@ package havener import ( - "context" "fmt" "io" + "os" "strings" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/gonvenience/term" - "github.com/gonvenience/text" - terminal "golang.org/x/term" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" "k8s.io/utils/pointer" + + "github.com/gonvenience/text" + "github.com/mattn/go-isatty" + "golang.org/x/term" ) +type NodeExecHelperPodConfig struct { + namespace string + podName string + + Annotations map[string]string + ContainerImage string + ContainerCmd []string + ContainerArgs []string + WaitTimeout time.Duration +} + +type ExecConfig struct { + Command []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + TTY bool +} + // PodExec executes the provided command in the referenced pod's container. -func (h *Hvnr) PodExec(pod *corev1.Pod, container string, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, tty bool) error { - logf(Verbose, "Executing command on pod: `%v`", strings.Join(command, " ")) +func (h *Hvnr) PodExec(pod *corev1.Pod, container string, execConfig ExecConfig) error { + logf(Verbose, "Executing command on pod: `%v`", strings.Join(execConfig.Command, " ")) req := h.client.CoreV1().RESTClient().Post(). Resource("pods"). @@ -50,11 +69,11 @@ func (h *Hvnr) PodExec(pod *corev1.Pod, container string, command []string, stdi SubResource("exec"). VersionedParams(&corev1.PodExecOptions{ Container: container, - Command: command, - Stdin: stdin != nil, - Stdout: stdout != nil, - Stderr: stderr != nil, - TTY: tty, + Command: execConfig.Command, + Stdin: execConfig.Stdin != nil, + Stdout: execConfig.Stdout != nil, + Stderr: execConfig.Stderr != nil, + TTY: execConfig.TTY, }, scheme.ParameterCodec) executor, err := remotecommand.NewSPDYExecutor(h.restconfig, "POST", req.URL()) @@ -63,7 +82,7 @@ func (h *Hvnr) PodExec(pod *corev1.Pod, container string, command []string, stdi } var tsq *terminalSizeQueue - if tty && term.IsTerminal() { + if execConfig.TTY && isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) { tsq = setupTerminalResizeWatcher() defer tsq.stop() } @@ -71,13 +90,23 @@ func (h *Hvnr) PodExec(pod *corev1.Pod, container string, command []string, stdi // Terminal needs to run in raw mode for the actual command execution when TTY is enabled. // The raw mode is the one where characters are not printed twice in the terminal. See // https://en.wikipedia.org/wiki/POSIX_terminal_interface#History for a bit more details. - if tty { - if stateToBeRestored, err := terminal.MakeRaw(0); err == nil { - defer func() { _ = terminal.Restore(0, stateToBeRestored) }() + if execConfig.TTY { + oldState, err := term.MakeRaw(0) + if err != nil { + return fmt.Errorf("failed to use raw terminal: %w", err) } + defer func() { _ = term.Restore(0, oldState) }() + } + + var streamOption = remotecommand.StreamOptions{ + Stdin: execConfig.Stdin, + Stdout: execConfig.Stdout, + Stderr: execConfig.Stderr, + Tty: execConfig.TTY, + TerminalSizeQueue: tsq, } - if err = executor.StreamWithContext(h.ctx, remotecommand.StreamOptions{Stdin: stdin, Stdout: stdout, Stderr: stderr, Tty: tty, TerminalSizeQueue: tsq}); err != nil { + if err = executor.StreamWithContext(h.ctx, streamOption); err != nil { return fmt.Errorf("failed to execute command on pod %s, container %s: %w", pod.Name, container, err) } @@ -86,42 +115,53 @@ func (h *Hvnr) PodExec(pod *corev1.Pod, container string, command []string, stdi } // NodeExec executes the provided command on the given node. -func (h *Hvnr) NodeExec(node corev1.Node, containerImage string, timeoutSeconds int, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, tty bool) error { - var ( - podName = text.RandomStringWithPrefix("node-exec-", 15) // unique pod name - namespace = "kube-system" - ) +func (h *Hvnr) NodeExec(node corev1.Node, hlpPodConfig NodeExecHelperPodConfig, execConfig ExecConfig) error { + hlpPodConfig.podName = text.RandomStringWithPrefix("node-exec-", 15) // unique pod name + hlpPodConfig.namespace = "kube-system" // Make sure to stop pod after command execution - defer func() { _ = h.PurgePod(namespace, podName, 10, metav1.DeletePropagationForeground) }() + defer func() { + _ = h.PurgePod(hlpPodConfig.namespace, hlpPodConfig.podName, 0, metav1.DeletePropagationBackground) + }() - pod, err := h.preparePodOnNode(node, namespace, podName, containerImage, timeoutSeconds, stdin != nil) + pod, err := h.preparePodOnNode(node, hlpPodConfig) if err != nil { return err } + // Unset the stderr in case TTY is set + // https://github.com/kubernetes/kubectl/blob/5b7c8b24b4361a97ab19de1d1e301a6c1bbaed1a/pkg/cmd/exec/exec.go#L370-L372 + if execConfig.TTY { + execConfig.Stderr = nil + } + // Execute command on pod and redirect output to users provided stdout and stderr - logf(Verbose, "Executing command on node: `%v`", strings.Join(command, " ")) + logf(Verbose, "Executing command on node: `%v`", strings.Join(execConfig.Command, " ")) + return h.PodExec( - pod, - "node-exec-container", - append([]string{"nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "--"}, command...), - stdin, - stdout, - stderr, - tty, + pod, "node-exec-container", + ExecConfig{ + Command: append([]string{"nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "--"}, execConfig.Command...), + Stdin: execConfig.Stdin, + Stdout: execConfig.Stdout, + Stderr: execConfig.Stderr, + TTY: execConfig.TTY, + }, ) } -func (h *Hvnr) preparePodOnNode(node corev1.Node, namespace string, name string, containerImage string, timeoutSeconds int, useStdin bool) (*corev1.Pod, error) { +func (h *Hvnr) preparePodOnNode(node corev1.Node, hlpPodConfig NodeExecHelperPodConfig) (*corev1.Pod, error) { // Add pod deletion to shutdown sequence list (in case of Ctrl+C exit) - AddShutdownFunction(func() { _ = h.PurgePod(namespace, name, 10, metav1.DeletePropagationBackground) }) + AddShutdownFunction(func() { + _ = h.PurgePod(hlpPodConfig.namespace, hlpPodConfig.podName, 10, metav1.DeletePropagationBackground) + }) // Pod configuration pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, + Name: hlpPodConfig.podName, + Namespace: hlpPodConfig.namespace, + Annotations: hlpPodConfig.Annotations, }, Spec: corev1.PodSpec{ NodeSelector: map[string]string{ @@ -135,9 +175,10 @@ func (h *Hvnr) preparePodOnNode(node corev1.Node, namespace string, name string, Containers: []corev1.Container{ { Name: "node-exec-container", - Image: containerImage, + Image: hlpPodConfig.ContainerImage, + Command: hlpPodConfig.ContainerCmd, + Args: hlpPodConfig.ContainerArgs, ImagePullPolicy: corev1.PullIfNotPresent, - Stdin: useStdin, SecurityContext: &corev1.SecurityContext{ Privileged: pointer.Bool(true), }, @@ -157,22 +198,22 @@ func (h *Hvnr) preparePodOnNode(node corev1.Node, namespace string, name string, } // Create pod in given namespace based on configuration - logf(Verbose, "Creating temporary pod *%s* in namespace *%s*", name, namespace) - pod, err := h.client.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + logf(Verbose, "Creating temporary pod _%s_/*%s*", hlpPodConfig.namespace, hlpPodConfig.podName) + pod, err := h.client.CoreV1().Pods(hlpPodConfig.namespace).Create(h.ctx, pod, metav1.CreateOptions{}) if err != nil { return nil, err } logf(Verbose, "Waiting for temporary pod to be started...") - if err := h.waitForPodReadiness(namespace, pod, timeoutSeconds); err != nil { + if err := h.waitForPodReadiness(pod, hlpPodConfig.WaitTimeout); err != nil { return nil, err } return pod, nil } -func (h *Hvnr) waitForPodReadiness(namespace string, pod *corev1.Pod, timeoutSeconds int) error { - watcher, err := h.client.CoreV1().Pods(namespace).Watch(context.TODO(), metav1.SingleObject(pod.ObjectMeta)) +func (h *Hvnr) waitForPodReadiness(pod *corev1.Pod, waitTimeout time.Duration) error { + watcher, err := h.client.CoreV1().Pods(pod.Namespace).Watch(h.ctx, metav1.SingleObject(pod.ObjectMeta)) if err != nil { return err } @@ -196,7 +237,7 @@ func (h *Hvnr) waitForPodReadiness(namespace string, pod *corev1.Pod, timeoutSec } }(watcher) - timeout := time.After(time.Duration(timeoutSeconds) * time.Second) + timeout := time.After(waitTimeout) for { select { @@ -209,10 +250,10 @@ func (h *Hvnr) waitForPodReadiness(namespace string, pod *corev1.Pod, timeoutSec description = "Unable to provide further details regarding the state of the pod." } - return fmt.Errorf("Giving up waiting for pod %s in namespace %s to become ready within %s: %w", + return fmt.Errorf("Giving up waiting for pod %s in namespace %s to become ready within %v: %w", pod.Name, pod.Namespace, - text.Plural(timeoutSeconds, "second"), + waitTimeout, fmt.Errorf("status of pod at the moment of the timeout:\n\n%s", description), ) } diff --git a/pkg/havener/logs.go b/pkg/havener/logs.go index e35c6a66..903014db 100644 --- a/pkg/havener/logs.go +++ b/pkg/havener/logs.go @@ -24,7 +24,6 @@ import ( "archive/tar" "bytes" "compress/gzip" - "context" "fmt" "io" "os" @@ -238,7 +237,7 @@ func (h *Hvnr) retrieveFilesFromPod(pod *corev1.Pod, baseDir string, findCommand for _, container := range pod.Spec.Containers { // Ignore all container that have no shell available - if err := h.PodExec(pod, container.Name, []string{"/bin/sh", "-c", "true"}, nil, io.Discard, nil, false); err != nil { + if err := h.PodExec(pod, container.Name, ExecConfig{Command: []string{"/bin/sh", "-c", "true"}, Stdout: io.Discard, Stderr: io.Discard}); err != nil { continue } @@ -253,15 +252,14 @@ func (h *Hvnr) retrieveFilesFromPod(pod *corev1.Pod, baseDir string, findCommand err := h.PodExec( pod, container.Name, - []string{"/bin/sh", "-c", - fmt.Sprintf( - retrieveScript, - strings.Join(findCommands, "; "), - )}, - nil, - write, - nil, - false, + ExecConfig{ + Command: []string{"/bin/sh", "-c", + fmt.Sprintf( + retrieveScript, + strings.Join(findCommands, "; "), + )}, + Stdout: write, + }, ) if err != nil { @@ -333,7 +331,7 @@ func (h *Hvnr) retrieveContainerLogs(pod *corev1.Pod, baseDir string) []error { errors := []error{} streamToFile := func(req *rest.Request, filename string) error { - readCloser, err := req.Stream(context.TODO()) + readCloser, err := req.Stream(h.ctx) if err != nil { return err } diff --git a/pkg/havener/top.go b/pkg/havener/top.go index 364b9274..a7fbb5b8 100644 --- a/pkg/havener/top.go +++ b/pkg/havener/top.go @@ -22,7 +22,6 @@ package havener import ( "bytes" - "context" "encoding/json" "fmt" "strconv" @@ -114,7 +113,7 @@ func (h *Hvnr) TopDetails() (*TopDetails, error) { Containers: map[string]map[string]map[string]ContainerDetails{}, } - nodeList, err := h.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + nodeList, err := h.client.CoreV1().Nodes().List(h.ctx, metav1.ListOptions{}) if err != nil { return nil, err } @@ -165,7 +164,7 @@ func (h *Hvnr) TopDetails() (*TopDetails, error) { go func() { defer wg.Done() - nodeMetricsJSON, err := h.client.CoreV1().RESTClient().Get().AbsPath("apis/metrics.k8s.io/v1beta1/nodes").DoRaw(context.TODO()) + nodeMetricsJSON, err := h.client.CoreV1().RESTClient().Get().AbsPath("apis/metrics.k8s.io/v1beta1/nodes").DoRaw(h.ctx) if err != nil { errChan <- err return @@ -191,7 +190,7 @@ func (h *Hvnr) TopDetails() (*TopDetails, error) { go func() { defer wg.Done() - podMetricsJSON, err := h.client.CoreV1().RESTClient().Get().AbsPath("apis/metrics.k8s.io/v1beta1/pods").DoRaw(context.TODO()) + podMetricsJSON, err := h.client.CoreV1().RESTClient().Get().AbsPath("apis/metrics.k8s.io/v1beta1/pods").DoRaw(h.ctx) if err != nil { errChan <- err return @@ -220,9 +219,9 @@ func (h *Hvnr) TopDetails() (*TopDetails, error) { defer wg.Done() podname := fmt.Sprintf("havener-usage-retriever-%s", node.Name) - pod, err := h.client.CoreV1().Pods("kube-system").Get(context.TODO(), podname, metav1.GetOptions{}) + pod, err := h.client.CoreV1().Pods("kube-system").Get(h.ctx, podname, metav1.GetOptions{}) if err != nil { - pod, err = h.preparePodOnNode(node, "kube-system", podname, "alpine", 5, true) + pod, err = h.preparePodOnNode(node, NodeExecHelperPodConfig{namespace: "kube-system", podName: podname, ContainerImage: "alpine", WaitTimeout: 5 * time.Second}) if err != nil { return } @@ -235,13 +234,13 @@ func (h *Hvnr) TopDetails() (*TopDetails, error) { var stdout bytes.Buffer var stderr bytes.Buffer err = h.PodExec( - pod, - "node-exec-container", - []string{"nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "--", "/bin/cat", "/proc/loadavg"}, - strings.NewReader(""), - &stdout, - &stderr, - false, + pod, "node-exec-container", + ExecConfig{ + Command: []string{"nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "--", "/bin/cat", "/proc/loadavg"}, + Stdout: &stdout, + Stderr: &stderr, + TTY: false, + }, ) if err != nil {