main: stacktrace/coredump on error

If the runtime detects an internal error or if it receives
a fatal signal, write a stack trace to the log and exit.

If debug is enabled also dump core.

Note: This replaces the previous `debug.SetTraceback()` call which
logs to `stderr`, which is not desirable.

Fixes #1053.

Signed-off-by: James O. D. Hunt <[email protected]>

config.go

@@ -447,11 +447,14 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		return "", config, err
 	}
 
-	if !tomlConf.Runtime.Debug {
+	if tomlConf.Runtime.Debug {
+		crashOnError = true
+	} else {
 		// If debug is not required, switch back to the original
 		// default log priority, otherwise continue in debug mode.
 		ccLog.Logger.Level = originalLoggerLevel
 	}
+
 	if tomlConf.Runtime.InterNetworkModel != "" {
 		err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel)
 		if err != nil {

fatal.go

@@ -0,0 +1,80 @@
+// Copyright 2018 Intel Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os/signal"
+	"runtime/pprof"
+	"strings"
+	"syscall"
+)
+
+// List of fatal signals
+var sigFatal = map[syscall.Signal]bool{
+	syscall.SIGABRT:   true,
+	syscall.SIGBUS:    true,
+	syscall.SIGILL:    true,
+	syscall.SIGQUIT:   true,
+	syscall.SIGSEGV:   true,
+	syscall.SIGSTKFLT: true,
+	syscall.SIGSYS:    true,
+	syscall.SIGTRAP:   true,
+}
+
+func handlePanic() {
+	r := recover()
+
+	if r != nil {
+		msg := fmt.Sprintf("%s", r)
+		ccLog.WithField("panic", msg).Error("fatal error")
+
+		die()
+	}
+}
+
+func backtrace() {
+	profiles := pprof.Profiles()
+
+	buf := &bytes.Buffer{}
+
+	for _, p := range profiles {
+		// The magic number requests a full stacktrace. See
+		// https://golang.org/pkg/runtime/pprof/#Profile.WriteTo.
+		pprof.Lookup(p.Name()).WriteTo(buf, 2)
+	}
+
+	for _, line := range strings.Split(buf.String(), "\n") {
+		ccLog.Error(line)
+	}
+}
+
+func fatalSignal(sig syscall.Signal) bool {
+	return sigFatal[sig]
+}
+
+func fatalSignals() []syscall.Signal {
+	var signals []syscall.Signal
+
+	for sig := range sigFatal {
+		signals = append(signals, sig)
+
+	}
+
+	return signals
+}
+
+func die() {
+	backtrace()
+
+	if crashOnError {
+		signal.Reset(syscall.SIGABRT)
+		syscall.Kill(0, syscall.SIGABRT)
+	}
+
+	exit(1)
+}

main.go

@@ -19,9 +19,10 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
 	goruntime "runtime"
-	"runtime/debug"
 	"strings"
+	"syscall"
 
 	vc "github.com/containers/virtcontainers"
 	"github.com/containers/virtcontainers/pkg/oci"
@@ -60,6 +61,9 @@ var ccLog *logrus.Entry
 // required.
 var originalLoggerLevel logrus.Level
 
+// if true, coredump when an internal error occurs or a fatal signal is received
+var crashOnError = false
+
 // concrete virtcontainer implementation
 var virtcontainersImpl = &vc.VCImpl{}
 
@@ -155,9 +159,26 @@ func init() {
 	// config file parsing, it is prudent to operate in verbose mode.
 	originalLoggerLevel = ccLog.Logger.Level
 	ccLog.Logger.Level = logrus.DebugLevel
+}
+
+func setupSignalHandler() {
+	sigCh := make(chan os.Signal, 8)
+
+	for _, sig := range fatalSignals() {
+		signal.Notify(sigCh, sig)
+	}
+
+	go func() {
+		sig := <-sigCh
 
-	// Force a coredump + full stacktrace on internal error
-	debug.SetTraceback("crash")
+		nativeSignal, ok := sig.(syscall.Signal)
+		if ok {
+			if fatalSignal(nativeSignal) {
+				ccLog.WithField("signal", sig).Error("received fatal signal")
+				die()
+			}
+		}
+	}()
 }
 
 // beforeSubcommands is the function to perform preliminary checks
@@ -349,6 +370,8 @@ func (f *fatalWriter) Write(p []byte) (n int, err error) {
 }
 
 func createRuntime() {
+	setupSignalHandler()
+
 	setCLIGlobals()
 
 	err := createRuntimeApp(os.Args)
@@ -358,5 +381,6 @@ func createRuntime() {
 }
 
 func main() {
+	defer handlePanic()
 	createRuntime()
 }