From cc597773ddcc6b8881028fa19f43d8d36ae8adb7 Mon Sep 17 00:00:00 2001 From: jharrod Date: Thu, 12 Sep 2024 09:36:36 -0600 Subject: [PATCH] Add node stage vol fault points * Add FIJI points for node stage vol --- frontend/csi/volume_publish_manager.go | 14 ++++++++++++++ utils/devices_linux.go | 21 +++++++++++++++++++++ utils/filesystem.go | 16 ++++++++++++++-- utils/iscsi.go | 22 ++++++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/frontend/csi/volume_publish_manager.go b/frontend/csi/volume_publish_manager.go index 8cd2a6281..f5f9d3b14 100644 --- a/frontend/csi/volume_publish_manager.go +++ b/frontend/csi/volume_publish_manager.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/afero" "github.com/netapp/trident/config" + "github.com/netapp/trident/internal/fiji" . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils" "github.com/netapp/trident/utils/errors" @@ -25,6 +26,9 @@ const volumePublishInfoFilename = "volumePublishInfo.json" var ( osFs = afero.NewOsFs() fileDeleter = utils.DeleteFile + + beforeWritingTempTrackingInfoFile = fiji.Register("beforeWritingTempTrackingInfoFile", "volume_publish_manager") + beforeUpdatingTrackingInfoFile = fiji.Register("beforeUpdatingTrackingInfoFile", "volume_publish_manager") ) type VolumePublishManager struct { @@ -64,6 +68,11 @@ func (v *VolumePublishManager) WriteTrackingInfo( // destination once it succeeds (a rename is just moving a filesystem pointer, not additional I/O). tmpFile = path.Join(v.volumeTrackingInfoPath, tmpFile) Logc(ctx).WithField("tempFile", tmpFile).Debug("Writing temporary tracking info file.") + + if err := beforeWritingTempTrackingInfoFile.Inject(); err != nil { + return err + } + err := utils.JsonReaderWriter.WriteJSONFile(ctx, trackingInfo, tmpFile, "volume tracking info") if err != nil { return err @@ -72,6 +81,11 @@ func (v *VolumePublishManager) WriteTrackingInfo( trackingFile := path.Join(v.volumeTrackingInfoPath, filename) Logc(ctx).WithField("fileName", trackingFile).Debug("Updating tracking info file.") + + if err := beforeUpdatingTrackingInfoFile.Inject(); err != nil { + return err + } + return osFs.Rename(tmpFile, trackingFile) } diff --git a/utils/devices_linux.go b/utils/devices_linux.go index 82ed69b74..df5234509 100644 --- a/utils/devices_linux.go +++ b/utils/devices_linux.go @@ -19,6 +19,7 @@ import ( "golang.org/x/net/context" "golang.org/x/sys/unix" + "github.com/netapp/trident/internal/fiji" . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils/errors" ) @@ -39,6 +40,12 @@ const ( cryptsetupIsLuksDeviceIsNotLuksStatusCode = 1 ) +var ( + beforeCryptSetupFormat = fiji.Register("beforeCryptSetupFormat", "node_server") + duringOpenBeforeCryptSetupOpen = fiji.Register("duringOpenBeforeCryptSetupOpen", "devices_linux") + duringRotatePassphraseBeforeLuksKeyChange = fiji.Register("duringRotatePassphraseBeforeLuksKeyChange", "devices_linux") +) + // flushOneDevice flushes any outstanding I/O to a disk func flushOneDevice(ctx context.Context, devicePath string) error { fields := LogFields{"rawDevicePath": devicePath} @@ -141,6 +148,11 @@ func (d *LUKSDevice) luksFormat(ctx context.Context, luksPassphrase string) erro device := d.RawDevicePath() Logc(ctx).WithField("device", device).Debug("Formatting LUKS device.") + + if err := beforeCryptSetupFormat.Inject(); err != nil { + return err + } + if output, err := command.ExecuteWithTimeoutAndInput( ctx, "cryptsetup", luksCommandTimeout, true, luksPassphrase, "luksFormat", device, "--type", "luks2", "-c", "aes-xts-plain64", @@ -204,6 +216,11 @@ func (d *LUKSDevice) Open(ctx context.Context, luksPassphrase string) error { fields := LogFields{"device": device, "luksDeviceName": luksDeviceName} Logc(ctx).WithFields(fields).Debug("Opening LUKS device.") + + if err := duringOpenBeforeCryptSetupOpen.Inject(); err != nil { + return err + } + output, err := command.ExecuteWithTimeoutAndInput( ctx, "cryptsetup", luksCommandTimeout, true, luksPassphrase, "open", device, luksDeviceName, "--type", "luks2", @@ -444,6 +461,10 @@ func (d *LUKSDevice) RotatePassphrase( // Rely on Linux's ability to reference already-open files with /dev/fd/* oldKeyFilename := fmt.Sprintf("/dev/fd/%d", fd) + if err = duringRotatePassphraseBeforeLuksKeyChange.Inject(); err != nil { + return err + } + output, err := command.ExecuteWithTimeoutAndInput( ctx, "cryptsetup", luksCommandTimeout, true, luksPassphrase, "luksChangeKey", "-d", oldKeyFilename, d.RawDevicePath(), diff --git a/utils/filesystem.go b/utils/filesystem.go index 8e53e1278..45fccfa14 100644 --- a/utils/filesystem.go +++ b/utils/filesystem.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/afero" "go.uber.org/multierr" + "github.com/netapp/trident/internal/fiji" . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils/errors" "github.com/netapp/trident/utils/models" @@ -44,6 +45,9 @@ const ( var ( osFs = afero.NewOsFs() JsonReaderWriter = NewJSONReaderWriter() + + duringFormatVolume = fiji.Register("duringFormatVolume", "filesystem") + duringRepairVolume = fiji.Register("duringRepairVolume", "filesystem") ) type jsonReaderWriter struct{} @@ -99,8 +103,12 @@ func formatVolume(ctx context.Context, device, fstype string) error { logFields := LogFields{"device": device, "fsType": fstype} Logc(ctx).WithFields(logFields).Debug(">>>> filesystem.formatVolume") defer Logc(ctx).WithFields(logFields).Debug("<<<< filesystem.formatVolume") - var err error + if err := duringFormatVolume.Inject(); err != nil { + return err + } + + var err error switch fstype { case fsXfs: _, err = command.Execute(ctx, "mkfs.xfs", "-f", device) @@ -153,8 +161,12 @@ func repairVolume(ctx context.Context, device, fstype string) { Logc(ctx).WithFields(logFields).Debug(">>>> filesystem.repairVolume") defer Logc(ctx).WithFields(logFields).Debug("<<<< filesystem.repairVolume") - var err error + // Note: FIJI error injection will have no effect due to no error return. Panic injection is still valid. + if err := duringRepairVolume.Inject(); err != nil { + Logc(ctx).WithError(err).Debug("FIJI error in repairVolume has no effect, no error return.") + } + var err error switch fstype { case "xfs": break // fsck.xfs does nothing diff --git a/utils/iscsi.go b/utils/iscsi.go index dfc5418a7..6a4b77df7 100644 --- a/utils/iscsi.go +++ b/utils/iscsi.go @@ -20,6 +20,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/netapp/trident/config" + "github.com/netapp/trident/internal/fiji" . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils/errors" "github.com/netapp/trident/utils/models" @@ -62,6 +63,11 @@ var ( // -<36 characters of trident version uuid> // ex: Kubernetes-NodeA-01-ad1b8212-8095-49a0-82d4-ef4f8b5b620z perNodeIgroupRegex = regexp.MustCompile(`^[0-9A-z\-.]{1,59}-[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}`) + + duringConfigureISCSITargetBeforeISCSIAdmUpdate = fiji.Register("duringConfigureISCSITargetBeforeISCSIAdmUpdate", "iscsi") + duringRescanOneLunBeforeFileWrite = fiji.Register("duringRescanOneLunBeforeFileWrite", "iscsi") + duringPurgeOneLunBeforeFileWrite = fiji.Register("duringPurgeOneLunBeforeFileWrite", "iscsi") + duringISCSIScanTargetLunAfterFileOpen = fiji.Register("duringISCSIScanTargetLunAfterFileOpen", "iscsi") ) // IsPerNodeIgroup accepts an igroup and returns whether that igroup matches the per-node igroup schema. @@ -919,6 +925,10 @@ func iSCSIScanTargetLUN(ctx context.Context, lunID int, hosts []int) error { return err } + if err = duringISCSIScanTargetLunAfterFileOpen.Inject(); err != nil { + return err + } + if written, err := f.WriteString(scanCmd); err != nil { Logc(ctx).WithFields(LogFields{"file": filename, "error": err}).Warning("Could not write to file.") f.Close() @@ -1001,6 +1011,10 @@ func purgeOneLun(ctx context.Context, path string) error { } defer f.Close() + if err = duringPurgeOneLunBeforeFileWrite.Inject(); err != nil { + return err + } + // Deleting a LUN is achieved by writing the string "1" to the "delete" file written, err := f.WriteString("1") if err != nil { @@ -1027,6 +1041,10 @@ func rescanOneLun(ctx context.Context, path string) error { } defer f.Close() + if err = duringRescanOneLunBeforeFileWrite.Inject(); err != nil { + return err + } + written, err := f.WriteString("1") if err != nil { Logc(ctx).WithFields(LogFields{"file": filename, "error": err}).Warning("Could not write to file.") @@ -1487,6 +1505,10 @@ func configureISCSITarget(ctx context.Context, iqn, portal, name, value string) }).Debug(">>>> iscsi.configureISCSITarget") defer Logc(ctx).Debug("<<<< iscsi.configureISCSITarget") + if err := duringConfigureISCSITargetBeforeISCSIAdmUpdate.Inject(); err != nil { + return err + } + args := []string{"-m", "node", "-T", iqn, "-p", formatPortal(portal), "-o", "update", "-n", name, "-v", value} if _, err := execIscsiadmCommand(ctx, args...); err != nil { Logc(ctx).WithField("error", err).Warn("Error configuring iSCSI target.")