Add timeout for command execution

Signed-off-by: Madhu Rajanna <[email protected]>

cmd/cephfs/main.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/ceph/ceph-csi/pkg/cephfs"
 	"github.com/ceph/ceph-csi/pkg/util"
+
 	"k8s.io/klog"
 )
 
@@ -35,6 +36,8 @@ var (
 )
 
 func main() {
+	flag.IntVar(&util.Timeout, "execTimeout", 30, "timeout value for exec commands")
+
 	util.InitLogging()
 
 	if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "controller")); err != nil {

cmd/rbd/main.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/ceph/ceph-csi/pkg/rbd"
 	"github.com/ceph/ceph-csi/pkg/util"
+
 	"k8s.io/klog"
 )
 
@@ -35,6 +36,8 @@ var (
 )
 
 func main() {
+	flag.IntVar(&util.Timeout, "execTimeout", 30, "timeout value for exec commands")
+
 	util.InitLogging()
 
 	if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil {

pkg/cephfs/cephuser.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/ceph/ceph-csi/pkg/util"
 	"k8s.io/klog"
 )
 
@@ -62,7 +63,7 @@ func getCephUser(adminCr *credentials, volID volumeID) (*cephEntity, error) {
 		"get", entityName,
 	}
 
-	out, err := execCommand("ceph", args[:]...)
+	out, err := util.ExecCommand("ceph", args[:]...)
 	if err != nil {
 		return nil, fmt.Errorf("cephfs: ceph failed with following error: %s\ncephfs: ceph output: %s", err, out)
 	}

pkg/cephfs/util.go

@@ -21,13 +21,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"os/exec"
 
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"github.com/ceph/ceph-csi/pkg/util"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"k8s.io/kubernetes/pkg/util/mount"
 )
 
@@ -37,15 +36,8 @@ func makeVolumeID(volName string) volumeID {
 	return volumeID("csi-cephfs-" + volName)
 }
 
-func execCommand(command string, args ...string) ([]byte, error) {
-	klog.V(4).Infof("cephfs: EXEC %s %s", command, args)
-
-	cmd := exec.Command(command, args...) // #nosec
-	return cmd.CombinedOutput()
-}
-
 func execCommandAndValidate(program string, args ...string) error {
-	out, err := execCommand(program, args...)
+	out, err := util.ExecCommand(program, args...)
 	if err != nil {
 		return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out)
 	}
@@ -55,7 +47,7 @@ func execCommandAndValidate(program string, args ...string) error {
 
 func execCommandJSON(v interface{}, args ...string) error {
 	program := "ceph"
-	out, err := execCommand(program, args...)
+	out, err := util.ExecCommand(program, args...)
 
 	if err != nil {
 		return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out)

pkg/cephfs/volumemounter.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+
+	"github.com/ceph/ceph-csi/pkg/util"
 )
 
 const (
@@ -111,7 +113,7 @@ func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions, vo
 		"-o", "nonempty",
 	}
 
-	out, err := execCommand("ceph-fuse", args[:]...)
+	out, err := util.ExecCommand("ceph-fuse", args[:]...)
 	if err != nil {
 		return fmt.Errorf("cephfs: ceph-fuse failed with following error: %s\ncephfs: ceph-fuse output: %s", err, out)
 	}

pkg/rbd/rbd_attach.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ceph/ceph-csi/pkg/util"
+
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog"
 )
@@ -211,12 +213,13 @@ func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string,
 
 // Check if rbd-nbd tools are installed.
 func checkRbdNbdTools() bool {
-	_, err := execCommand("modprobe", []string{"nbd"})
+
+	_, err := util.ExecCommand("modprobe", []string{"nbd"}...)
 	if err != nil {
 		klog.V(3).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
 		return false
 	}
-	if _, err := execCommand(rbdTonbd, []string{"--version"}); err != nil {
+	if _, err := util.ExecCommand(rbdTonbd, []string{"--version"}...); err != nil {
 		klog.V(3).Infof("rbd-nbd: running rbd-nbd --version failed with error %v", err)
 		return false
 	}
@@ -247,7 +250,7 @@ func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string
 			}
 		}()
 
-		_, err = execCommand("modprobe", []string{moduleName})
+		_, err = util.ExecCommand("modprobe", []string{moduleName}...)
 		if err != nil {
 			klog.Warningf("rbd: failed to load rbd kernel module:%v", err)
 			return "", err
@@ -291,8 +294,8 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri
 		cmdName = rbdTonbd
 	}
 
-	output, err := execCommand(cmdName, []string{
-		"map", imagePath, "--id", userID, "-m", mon, "--key=" + key})
+	output, err := util.ExecCommand(cmdName, []string{
+		"map", imagePath, "--id", userID, "-m", mon, "--key=" + key}...)
 	if err != nil {
 		klog.Warningf("rbd: map error %v, rbd output: %s", err, string(output))
 		return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
@@ -334,7 +337,7 @@ func detachRBDDevice(devicePath string) error {
 		cmdName = rbdTonbd
 	}
 
-	output, err = execCommand(cmdName, []string{"unmap", devicePath})
+	output, err = util.ExecCommand(cmdName, []string{"unmap", devicePath}...)
 	if err != nil {
 		return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output))
 	}

pkg/rbd/rbd_util.go

@@ -22,6 +22,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ceph/ceph-csi/pkg/util"
+
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog"
@@ -135,7 +137,7 @@ func createRBDImage(pOpts *rbdVolume, volSz int, adminID string, credentials map
 	if pOpts.ImageFormat == rbdImageFormat2 {
 		args = append(args, "--image-feature", pOpts.ImageFeatures)
 	}
-	output, err = execCommand("rbd", args)
+	output, err = util.ExecCommand("rbd", args...)
 
 	if err != nil {
 		return errors.Wrapf(err, "failed to create rbd image, command output: %s", string(output))
@@ -165,7 +167,7 @@ func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) (
 
 	klog.V(4).Infof("rbd: status %s using mon %s, pool %s", image, mon, pOpts.Pool)
 	args := []string{"status", image, "--pool", pOpts.Pool, "-m", mon, "--id", userID, "--key=" + key}
-	cmd, err = execCommand("rbd", args)
+	cmd, err = util.ExecCommand("rbd", args...)
 	output = string(cmd)
 
 	if err, ok := err.(*exec.Error); ok {
@@ -212,20 +214,14 @@ func deleteRBDImage(pOpts *rbdVolume, adminID string, credentials map[string]str
 
 	klog.V(4).Infof("rbd: rm %s using mon %s, pool %s", image, mon, pOpts.Pool)
 	args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminID, "-m", mon, "--key=" + key}
-	output, err = execCommand("rbd", args)
+	output, err = util.ExecCommand("rbd", args...)
 	if err == nil {
 		return nil
 	}
 	klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output))
 	return err
 }
 
-func execCommand(command string, args []string) ([]byte, error) {
-	// #nosec
-	cmd := exec.Command(command, args...)
-	return cmd.CombinedOutput()
-}
-
 func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
 	var ok bool
 	rbdVol := &rbdVolume{}
@@ -374,7 +370,7 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]
 	klog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s ", image, mon, pOpts.Pool)
 	args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key}
 
-	output, err = execCommand("rbd", args)
+	output, err = util.ExecCommand("rbd", args...)
 
 	if err != nil {
 		return errors.Wrapf(err, "failed to protect snapshot, command output: %s", string(output))
@@ -401,7 +397,7 @@ func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s
 	klog.V(4).Infof("rbd: snap create %s using mon %s, pool %s", image, mon, pOpts.Pool)
 	args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key}
 
-	output, err = execCommand("rbd", args)
+	output, err = util.ExecCommand("rbd", args...)
 
 	if err != nil {
 		return errors.Wrapf(err, "failed to create snapshot, command output: %s", string(output))
@@ -428,8 +424,7 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[strin
 	klog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s", image, mon, pOpts.Pool)
 	args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key}
 
-	output, err = execCommand("rbd", args)
-
+	output, err = util.ExecCommand("rbd", args...)
 	if err != nil {
 		return errors.Wrapf(err, "failed to unprotect snapshot, command output: %s", string(output))
 	}
@@ -455,7 +450,7 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s
 	klog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s", image, mon, pOpts.Pool)
 	args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key}
 
-	output, err = execCommand("rbd", args)
+	output, err = util.ExecCommand("rbd", args...)
 
 	if err != nil {
 		return errors.Wrapf(err, "failed to delete snapshot, command output: %s", string(output))
@@ -482,7 +477,7 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string
 	klog.V(4).Infof("rbd: clone %s using mon %s, pool %s", image, mon, pVolOpts.Pool)
 	args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", adminID, "-m", mon, "--key=" + key}
 
-	output, err = execCommand("rbd", args)
+	output, err = util.ExecCommand("rbd", args...)
 
 	if err != nil {
 		return errors.Wrapf(err, "failed to restore snapshot, command output: %s", string(output))

pkg/util/utils.go

@@ -0,0 +1,32 @@
+package util
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"time"
+
+	"k8s.io/klog"
+)
+
+// Timeout for Command execution
+// exporting Timeout to make it configurable
+var Timeout int
+
+// ExecCommand is a wrapper over exec.Command with timeout
+func ExecCommand(command string, args ...string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Timeout)*time.Second)
+	defer cancel()
+	klog.V(4).Infof("exec %s %s", command, args)
+
+	cmd := exec.Command(command, args...) // #nosec
+	out, err := cmd.CombinedOutput()
+	if ctx.Err() == context.DeadlineExceeded {
+		return nil, fmt.Errorf("command timed out")
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("command exited with non-zero code: %v", err)
+	}
+	return out, err
+}