Add node stage vol fault points

* Add FIJI points for node stage vol

frontend/csi/volume_publish_manager.go

@@ -14,6 +14,7 @@ import (
 	"github.com/spf13/afero"
 
 	"github.com/netapp/trident/config"
+	"github.com/netapp/trident/internal/fiji"
 	. "github.com/netapp/trident/logging"
 	"github.com/netapp/trident/utils"
 	"github.com/netapp/trident/utils/errors"
@@ -25,6 +26,9 @@ const volumePublishInfoFilename = "volumePublishInfo.json"
 var (
 	osFs        = afero.NewOsFs()
 	fileDeleter = utils.DeleteFile
+
+	beforeWritingTempTrackingInfoFile = fiji.Register("beforeWritingTempTrackingInfoFile", "volume_publish_manager")
+	beforeUpdatingTrackingInfoFile    = fiji.Register("beforeUpdatingTrackingInfoFile", "volume_publish_manager")
 )
 
 type VolumePublishManager struct {
@@ -64,6 +68,11 @@ func (v *VolumePublishManager) WriteTrackingInfo(
 	// destination once it succeeds (a rename is just moving a filesystem pointer, not additional I/O).
 	tmpFile = path.Join(v.volumeTrackingInfoPath, tmpFile)
 	Logc(ctx).WithField("tempFile", tmpFile).Debug("Writing temporary tracking info file.")
+
+	if err := beforeWritingTempTrackingInfoFile.Inject(); err != nil {
+		return err
+	}
+
 	err := utils.JsonReaderWriter.WriteJSONFile(ctx, trackingInfo, tmpFile, "volume tracking info")
 	if err != nil {
 		return err
@@ -72,6 +81,11 @@ func (v *VolumePublishManager) WriteTrackingInfo(
 	trackingFile := path.Join(v.volumeTrackingInfoPath, filename)
 
 	Logc(ctx).WithField("fileName", trackingFile).Debug("Updating tracking info file.")
+
+	if err := beforeUpdatingTrackingInfoFile.Inject(); err != nil {
+		return err
+	}
+
 	return osFs.Rename(tmpFile, trackingFile)
 }
 

utils/devices_linux.go

@@ -19,6 +19,7 @@ import (
 	"golang.org/x/net/context"
 	"golang.org/x/sys/unix"
 
+	"github.com/netapp/trident/internal/fiji"
 	. "github.com/netapp/trident/logging"
 	"github.com/netapp/trident/utils/errors"
 )
@@ -39,6 +40,12 @@ const (
 	cryptsetupIsLuksDeviceIsNotLuksStatusCode = 1
 )
 
+var (
+	beforeCryptSetupFormat                    = fiji.Register("beforeCryptSetupFormat", "node_server")
+	duringOpenBeforeCryptSetupOpen            = fiji.Register("duringOpenBeforeCryptSetupOpen", "devices_linux")
+	duringRotatePassphraseBeforeLuksKeyChange = fiji.Register("duringRotatePassphraseBeforeLuksKeyChange", "devices_linux")
+)
+
 // flushOneDevice flushes any outstanding I/O to a disk
 func flushOneDevice(ctx context.Context, devicePath string) error {
 	fields := LogFields{"rawDevicePath": devicePath}
@@ -141,6 +148,11 @@ func (d *LUKSDevice) luksFormat(ctx context.Context, luksPassphrase string) erro
 	device := d.RawDevicePath()
 
 	Logc(ctx).WithField("device", device).Debug("Formatting LUKS device.")
+
+	if err := beforeCryptSetupFormat.Inject(); err != nil {
+		return err
+	}
+
 	if output, err := command.ExecuteWithTimeoutAndInput(
 		ctx, "cryptsetup", luksCommandTimeout, true, luksPassphrase, "luksFormat", device,
 		"--type", "luks2", "-c", "aes-xts-plain64",
@@ -204,6 +216,11 @@ func (d *LUKSDevice) Open(ctx context.Context, luksPassphrase string) error {
 
 	fields := LogFields{"device": device, "luksDeviceName": luksDeviceName}
 	Logc(ctx).WithFields(fields).Debug("Opening LUKS device.")
+
+	if err := duringOpenBeforeCryptSetupOpen.Inject(); err != nil {
+		return err
+	}
+
 	output, err := command.ExecuteWithTimeoutAndInput(
 		ctx, "cryptsetup", luksCommandTimeout, true, luksPassphrase, "open", device,
 		luksDeviceName, "--type", "luks2",
@@ -444,6 +461,10 @@ func (d *LUKSDevice) RotatePassphrase(
 	// Rely on Linux's ability to reference already-open files with /dev/fd/*
 	oldKeyFilename := fmt.Sprintf("/dev/fd/%d", fd)
 
+	if err = duringRotatePassphraseBeforeLuksKeyChange.Inject(); err != nil {
+		return err
+	}
+
 	output, err := command.ExecuteWithTimeoutAndInput(
 		ctx, "cryptsetup", luksCommandTimeout, true, luksPassphrase, "luksChangeKey", "-d",
 		oldKeyFilename, d.RawDevicePath(),

utils/filesystem.go

@@ -16,6 +16,7 @@ import (
 	"github.com/spf13/afero"
 	"go.uber.org/multierr"
 
+	"github.com/netapp/trident/internal/fiji"
 	. "github.com/netapp/trident/logging"
 	"github.com/netapp/trident/utils/errors"
 	"github.com/netapp/trident/utils/models"
@@ -44,6 +45,9 @@ const (
 var (
 	osFs             = afero.NewOsFs()
 	JsonReaderWriter = NewJSONReaderWriter()
+
+	duringFormatVolume = fiji.Register("duringFormatVolume", "filesystem")
+	duringRepairVolume = fiji.Register("duringRepairVolume", "filesystem")
 )
 
 type jsonReaderWriter struct{}
@@ -99,8 +103,12 @@ func formatVolume(ctx context.Context, device, fstype string) error {
 	logFields := LogFields{"device": device, "fsType": fstype}
 	Logc(ctx).WithFields(logFields).Debug(">>>> filesystem.formatVolume")
 	defer Logc(ctx).WithFields(logFields).Debug("<<<< filesystem.formatVolume")
-	var err error
 
+	if err := duringFormatVolume.Inject(); err != nil {
+		return err
+	}
+
+	var err error
 	switch fstype {
 	case fsXfs:
 		_, err = command.Execute(ctx, "mkfs.xfs", "-f", device)
@@ -153,8 +161,12 @@ func repairVolume(ctx context.Context, device, fstype string) {
 	Logc(ctx).WithFields(logFields).Debug(">>>> filesystem.repairVolume")
 	defer Logc(ctx).WithFields(logFields).Debug("<<<< filesystem.repairVolume")
 
-	var err error
+	// Note: FIJI error injection will have no effect due to no error return. Panic injection is still valid.
+	if err := duringRepairVolume.Inject(); err != nil {
+		Logc(ctx).WithError(err).Debug("FIJI error in repairVolume has no effect, no error return.")
+	}
 
+	var err error
 	switch fstype {
 	case "xfs":
 		break // fsck.xfs does nothing

utils/iscsi.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cenkalti/backoff/v4"
 
 	"github.com/netapp/trident/config"
+	"github.com/netapp/trident/internal/fiji"
 	. "github.com/netapp/trident/logging"
 	"github.com/netapp/trident/utils/errors"
 	"github.com/netapp/trident/utils/models"
@@ -62,6 +63,11 @@ var (
 	// <up to and including 59 characters of a container orchestrator node name>-<36 characters of trident version uuid>
 	// ex: Kubernetes-NodeA-01-ad1b8212-8095-49a0-82d4-ef4f8b5b620z
 	perNodeIgroupRegex = regexp.MustCompile(`^[0-9A-z\-.]{1,59}-[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}`)
+
+	duringConfigureISCSITargetBeforeISCSIAdmUpdate = fiji.Register("duringConfigureISCSITargetBeforeISCSIAdmUpdate", "iscsi")
+	duringRescanOneLunBeforeFileWrite              = fiji.Register("duringRescanOneLunBeforeFileWrite", "iscsi")
+	duringPurgeOneLunBeforeFileWrite               = fiji.Register("duringPurgeOneLunBeforeFileWrite", "iscsi")
+	duringISCSIScanTargetLunAfterFileOpen          = fiji.Register("duringISCSIScanTargetLunAfterFileOpen", "iscsi")
 )
 
 // IsPerNodeIgroup accepts an igroup and returns whether that igroup matches the per-node igroup schema.
@@ -919,6 +925,10 @@ func iSCSIScanTargetLUN(ctx context.Context, lunID int, hosts []int) error {
 			return err
 		}
 
+		if err = duringISCSIScanTargetLunAfterFileOpen.Inject(); err != nil {
+			return err
+		}
+
 		if written, err := f.WriteString(scanCmd); err != nil {
 			Logc(ctx).WithFields(LogFields{"file": filename, "error": err}).Warning("Could not write to file.")
 			f.Close()
@@ -1001,6 +1011,10 @@ func purgeOneLun(ctx context.Context, path string) error {
 	}
 	defer f.Close()
 
+	if err = duringPurgeOneLunBeforeFileWrite.Inject(); err != nil {
+		return err
+	}
+
 	// Deleting a LUN is achieved by writing the string "1" to the "delete" file
 	written, err := f.WriteString("1")
 	if err != nil {
@@ -1027,6 +1041,10 @@ func rescanOneLun(ctx context.Context, path string) error {
 	}
 	defer f.Close()
 
+	if err = duringRescanOneLunBeforeFileWrite.Inject(); err != nil {
+		return err
+	}
+
 	written, err := f.WriteString("1")
 	if err != nil {
 		Logc(ctx).WithFields(LogFields{"file": filename, "error": err}).Warning("Could not write to file.")
@@ -1487,6 +1505,10 @@ func configureISCSITarget(ctx context.Context, iqn, portal, name, value string)
 	}).Debug(">>>> iscsi.configureISCSITarget")
 	defer Logc(ctx).Debug("<<<< iscsi.configureISCSITarget")
 
+	if err := duringConfigureISCSITargetBeforeISCSIAdmUpdate.Inject(); err != nil {
+		return err
+	}
+
 	args := []string{"-m", "node", "-T", iqn, "-p", formatPortal(portal), "-o", "update", "-n", name, "-v", value}
 	if _, err := execIscsiadmCommand(ctx, args...); err != nil {
 		Logc(ctx).WithField("error", err).Warn("Error configuring iSCSI target.")