e2e: interrupt VPP process if it becomes unresponsive (#281)

In case if VPP process hangs, try sending it SIGINT to see the stack trace in GDB
travelping · Jun 17, 2022 · ce29a9d · ce29a9d
1 parent 7bafeef
commit ce29a9d
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 1 deletion.
diff --git a/test/e2e/go.mod b/test/e2e/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/containernetworking/plugins v0.8.7
 	github.com/go-ping/ping v0.0.0-20201022122018-3977ed72668a
 	github.com/google/gopacket v1.1.18
+	github.com/mitchellh/go-ps v1.0.0
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
 	github.com/pkg/errors v0.9.1

diff --git a/test/e2e/go.sum b/test/e2e/go.sum
@@ -276,6 +276,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=

diff --git a/test/e2e/vpp/vpp.go b/test/e2e/vpp/vpp.go
@@ -30,6 +30,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,6 +40,7 @@ import (
 	"git.fd.io/govpp.git/api"
 	"git.fd.io/govpp.git/binapi/vpe"
 	"git.fd.io/govpp.git/core"
+	ps "github.com/mitchellh/go-ps"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"gopkg.in/tomb.v2"
@@ -239,6 +241,32 @@ func (vi *VPPInstance) prepareCommand() (*exec.Cmd, error) {
 	return exec.Command(NSENTER_CMD, args...), nil
 }
 
+func (vi *VPPInstance) InterruptVPP() {
+	if vi.cmd == nil || vi.cmd.ProcessState != nil {
+		vi.log.Info("can't interrupt VPP as it is not running")
+		return
+	}
+
+	if !vi.startupCfg.UseGDB {
+		vi.cmd.Process.Signal(syscall.SIGINT)
+		return
+	}
+
+	pl, err := ps.Processes()
+	if err != nil {
+		vi.log.Error(err, "error listing processes")
+	}
+	for _, proc := range pl {
+		if proc.PPid() == vi.cmd.Process.Pid {
+			vi.log.Info("forcibly interrupting VPP process",
+				"pid", proc.Pid(), "executable", proc.Executable())
+			if err := syscall.Kill(proc.Pid(), syscall.SIGINT); err != nil {
+				vi.log.Error(err, "error interrupting VPP process")
+			}
+		}
+	}
+}
+
 func (vi *VPPInstance) StartVPP() error {
 	vi.log.WithFields(logrus.Fields{
 		"cliSocket": vi.startupCfg.CLISock,
@@ -347,7 +375,9 @@ func (vi *VPPInstance) run(sigchldCh chan os.Signal, conev chan core.ConnectionE
 			}
 			return nil
 		case e := <-conev:
-			if e.State == core.Failed {
+			if e.State == core.Failed || e.State == core.NotResponding {
+				vi.log.Errorf("VPP conn failed / VPP not responding, interrupting the VPP process")
+				vi.InterruptVPP()
 				return errors.New("VPP API connection failed")
 			}
 		}