chore: Revert changes in #723 (#744)

determined-ai · Apr 18, 2024 · 8db94b4 · 8db94b4
1 parent 44216c3
commit 8db94b4
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 19 deletions.
diff --git a/master/internal/rm/dispatcherrm/dispatcher_resource_manager.go b/master/internal/rm/dispatcherrm/dispatcher_resource_manager.go
@@ -48,7 +48,7 @@ const (
 	pbsSchedulerType       = "pbs"
 	slurmResourcesCarrier  = "com.cray.analytics.capsules.carriers.hpc.slurm.SlurmResources"
 	pbsResourcesCarrier    = "com.cray.analytics.capsules.carriers.hpc.pbs.PbsResources"
-	launcherMinimumVersion = "3.2.3"
+	launcherMinimumVersion = "3.2.4"
 	root                   = "root"
 )
 

diff --git a/master/pkg/tasks/dispatcher_task.go b/master/pkg/tasks/dispatcher_task.go
@@ -219,21 +219,6 @@ func (t *TaskSpec) ToDispatcherManifest(
 	slurmArgs = append(slurmArgs, t.TaskContainerDefaults.Slurm.SbatchArgs()...)
 	slurmArgs = append(slurmArgs, t.SlurmConfig.SbatchArgs()...)
 
-	// SLURM can requeue a job if there are node level settings to specify it to do so.
-	// So, we have to explicitly specify NO_REQUEUE option to disable the requeueing of slurm jobs.
-	// Determined will manage the failed/preempted experiments by itself.
-	// In case, the user has already provided the NO_REQUEUE option, skip this step.
-	noRequeueExists := false
-	for _, arg := range slurmArgs {
-		if arg == "--no-requeue" {
-			noRequeueExists = true
-			break
-		}
-	}
-	if !noRequeueExists {
-		slurmArgs = append(slurmArgs, "--no-requeue")
-	}
-
 	logrus.Debugf("Custom slurm arguments: %s", slurmArgs)
 	errList := ValidateSlurm(slurmArgs)
 	if len(errList) > 0 {

diff --git a/master/pkg/tasks/dispatcher_task_test.go b/master/pkg/tasks/dispatcher_task_test.go
@@ -583,7 +583,7 @@ func Test_ToDispatcherManifest(t *testing.T) {
 			containerRunType: "singularity",
 			slotType:         device.CUDA,
 			Slurm:            []string{"--want=slurmArgs", "--X=Y"},
-			wantSlurmArgs:    []string{"--want=slurmArgs", "--X=Y", "--no-requeue"},
+			wantSlurmArgs:    []string{"--want=slurmArgs", "--X=Y"},
 		},
 		{
 			name:             "Test custom pbsArgs",
@@ -635,7 +635,24 @@ func Test_ToDispatcherManifest(t *testing.T) {
 			containerRunType: "singularity",
 			slotType:         device.CUDA,
 			Slurm:            []string{"--no-requeue"},
-			wantSlurmArgs:    []string{"--no-requeue"},
+			wantErr:          true,
+			errorContains:    "is not configurable",
+		},
+		{
+			name:             "Invalid PBS Option -r",
+			containerRunType: "singularity",
+			slotType:         device.CUDA,
+			Pbs:              []string{"-r"},
+			wantErr:          true,
+			errorContains:    "is not configurable",
+		},
+		{
+			name:             "Existing PBS Option -r n",
+			containerRunType: "singularity",
+			slotType:         device.CUDA,
+			Pbs:              []string{"-r n"},
+			wantErr:          true,
+			errorContains:    "is not configurable",
 		},
 	}
 
@@ -715,7 +732,7 @@ func Test_ToDispatcherManifest(t *testing.T) {
 				if len(tt.wantSlurmArgs) > 0 {
 					assert.DeepEqual(t, customs["slurmArgs"], tt.wantSlurmArgs)
 				} else {
-					assert.DeepEqual(t, customs["slurmArgs"], []string{"--no-requeue"})
+					assert.Assert(t, customs["slurmArgs"] == nil)
 				}
 
 				if len(tt.wantPbsArgs) > 0 {

diff --git a/master/pkg/tasks/dispatcher_verification.go b/master/pkg/tasks/dispatcher_verification.go
@@ -35,6 +35,7 @@ func ValidateSlurm(slurmOptions []string) []error {
 		"--error=", "-e",
 		"--output=", "-o",
 		"--partition=", "-p",
+		"--no-requeue",
 		"--requeue",
 	}
 	errors := validateWlmOptions(wlmSlurm, slurmOptions, forbiddenArgs)