feat: support hardware watchdog timers

Only enabled when activated by config, disabled on shutdown/reboot

Fixes siderolabs#8284

Signed-off-by: Dmitry Sharshakov <[email protected]>

internal/app/machined/pkg/controllers/runtime/watchdog_timer.go

@@ -0,0 +1,127 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"github.com/cosi-project/runtime/pkg/controller"
+	"github.com/cosi-project/runtime/pkg/safe"
+	"github.com/cosi-project/runtime/pkg/state"
+	"github.com/siderolabs/gen/optional"
+	"go.uber.org/zap"
+	"golang.org/x/sys/unix"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+
+	"github.com/siderolabs/talos/pkg/machinery/resources/config"
+)
+
+// WatchdogTimerController watches v1alpha1.Config, creates/updates/deletes kernel module specs.
+type WatchdogTimerController struct{}
+
+// Name implements controller.Controller interface.
+func (ctrl *WatchdogTimerController) Name() string {
+	return "runtime.WatchdogTimerController"
+}
+
+// Inputs implements controller.Controller interface.
+func (ctrl *WatchdogTimerController) Inputs() []controller.Input {
+	return []controller.Input{
+		{
+			Namespace: config.NamespaceName,
+			Type:      config.MachineConfigType,
+			ID:        optional.Some(config.V1Alpha1ID),
+		},
+	}
+}
+
+// Outputs implements controller.Controller interface.
+func (ctrl *WatchdogTimerController) Outputs() []controller.Output {
+	return nil
+}
+
+// Run implements controller.Controller interface.
+//
+//nolint:gocyclo
+func (ctrl *WatchdogTimerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	var wd *os.File
+
+	wdClose := func() {
+		logger.Info("Closing hardware watchdog", zap.String("filename", wd.Name()))
+		// Magic close: make sure old watchdog won't trip after we switch
+		wd.Write([]byte("V"))
+		wd.Close()
+		wd = nil
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			if wd != nil {
+				if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, wd.Fd(), unix.WDIOC_KEEPALIVE, 0); err != 0 {
+					return fmt.Errorf("Failed to feed watchdog: %w", err)
+				}
+			}
+			continue
+		case <-r.EventCh():
+		}
+
+		r.ResetRestartBackoff()
+
+		cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID)
+		if err != nil {
+			if !state.IsNotFoundError(err) {
+				return fmt.Errorf("Error getting watchdog config: %w", err)
+			}
+		}
+
+		if cfg != nil && cfg.Config().Machine() != nil {
+			dev := cfg.Config().Machine().Kernel().WatchdogDevice()
+			if dev == "" {
+				continue
+			}
+
+			timeout := cfg.Config().Machine().Kernel().WatchdogTimeout()
+			if timeout == 0 {
+				continue
+			}
+
+			// Close the watchdog if requested to use new one
+			if wd != nil && wd.Name() != dev {
+				wdClose()
+			}
+
+			if wd == nil {
+				wd, err = os.OpenFile(dev, syscall.O_RDWR, 0600)
+				if err != nil {
+					return errors.Errorf("Failed to open watchdog device: %s", err)
+				}
+				logger.Info("Opened hardware watchdog", zap.String("filename", dev))
+				// TODO: support reboot/shutdown watchdogs
+				defer wdClose()
+			}
+
+			if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, wd.Fd(), uintptr(unix.WDIOC_SETTIMEOUT), uintptr(unsafe.Pointer(&timeout))); err != 0 {
+				return fmt.Errorf("Failed to set watchdog timeout: %w", err)
+			}
+
+			// 3 pings per timeout should suffice in any case
+			ticker = time.NewTicker(time.Duration(timeout/3) * time.Second)
+			if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, wd.Fd(), uintptr(unix.WDIOC_KEEPALIVE), uintptr(0)); err != 0 {
+				return fmt.Errorf("Failed to feed watchdog: %w", err)
+			}
+			logger.Info("Set hardware watchdog timeout", zap.Int("timeout", timeout), zap.Int("feed_interval", timeout/3))
+		}
+	}
+}

internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go

@@ -299,6 +299,7 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error
 			V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(),
 		},
 		runtimecontrollers.NewUniqueMachineTokenController(),
+		&runtimecontrollers.WatchdogTimerController{},
 		&secrets.APICertSANsController{},
 		&secrets.APIController{},
 		&secrets.EtcdController{},

pkg/machinery/config/config/machine.go

@@ -450,6 +450,8 @@ type LoggingDestination interface {
 // Kernel describes Talos Linux kernel configuration.
 type Kernel interface {
 	Modules() []KernelModule
+	WatchdogDevice() string
+	WatchdogTimeout() int
 }
 
 // KernelModule describes Linux module to load.

pkg/machinery/config/types/v1alpha1/v1alpha1_kernel.go

@@ -15,6 +15,20 @@ func (kc *KernelConfig) Modules() []config.KernelModule {
 	return xslices.Map(kc.KernelModules, func(kmc *KernelModuleConfig) config.KernelModule { return kmc })
 }
 
+// WatchdogDevice implements config.Kernel interface.
+func (kc *KernelConfig) WatchdogDevice() string {
+	if kc.WatchdogNode == "" {
+		return "/dev/watchdog0"
+	}
+
+	return kc.WatchdogNode
+}
+
+// WatchdogTimeout implements config.Kernel interface.
+func (kc *KernelConfig) WatchdogTimeout() int {
+	return kc.WatchdogTime
+}
+
 // Name implements config.KernelModule interface.
 func (kmc *KernelModuleConfig) Name() string {
 	return kmc.ModuleName

pkg/machinery/config/types/v1alpha1/v1alpha1_types.go

@@ -2377,6 +2377,12 @@ type KernelConfig struct {
 	// description: |
 	//   Kernel modules to load.
 	KernelModules []*KernelModuleConfig `yaml:"modules,omitempty"`
+	// description: |
+	//   Watchdog device node to use.
+	WatchdogNode string `yaml:"watchdogDevice,omitempty"`
+	// description: |
+	//   Watchdog timeout value.
+	WatchdogTime int `yaml:"watchdogTimeout,omitempty"`
 }
 
 // KernelModuleConfig struct configures Linux kernel modules to load.

pkg/provision/providers/qemu/launch.go

@@ -277,6 +277,10 @@ func launchVM(config *LaunchConfig) error {
 		"virtio-serial",
 		"-device",
 		"virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
+		"-device",
+		"i6300esb,id=watchdog0",
+		"-watchdog-action",
+		"pause",
 	}
 
 	for i, disk := range config.DiskPaths {