Improve demystifying GKE spot node preemtion #patch (#354)

* Improve demystifying GKE spot node preemtion Signed-off-by: Bernhard Stadlbauer <[email protected]> * Add test Signed-off-by: Bernhard Stadlbauer <[email protected]> --------- Signed-off-by: Bernhard Stadlbauer <[email protected]>
flyteorg · May 26, 2023 · c499a48 · c499a48
1 parent 8a2f8ca
commit c499a48
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/go/tasks/pluginmachinery/flytek8s/pod_helper.go
@@ -622,7 +622,9 @@ func DemystifyFailure(status v1.PodStatus, info pluginsCore.TaskInfo) (pluginsCo
 	//         startTime: "2022-01-30T14:24:07Z"
 	//     }
 	// }
-	if code == "Shutdown" {
+	//
+	// In some versions of GKE the reason can also be "Terminated"
+	if code == "Shutdown" || code == "Terminated" {
 		return pluginsCore.PhaseInfoSystemRetryableFailure(Interrupted, message, &info), nil
 	}
 

diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go
@@ -900,6 +900,17 @@ func TestDemystifyFailure(t *testing.T) {
 		assert.Equal(t, "Interrupted", phaseInfo.Err().Code)
 		assert.Equal(t, core.ExecutionError_SYSTEM, phaseInfo.Err().Kind)
 	})
+
+	t.Run("GKE kubelet graceful node shutdown", func(t *testing.T) {
+		phaseInfo, err := DemystifyFailure(v1.PodStatus{
+			Message: "Foobar",
+			Reason:  "Terminated",
+		}, pluginsCore.TaskInfo{})
+		assert.Nil(t, err)
+		assert.Equal(t, pluginsCore.PhaseRetryableFailure, phaseInfo.Phase())
+		assert.Equal(t, "Interrupted", phaseInfo.Err().Code)
+		assert.Equal(t, core.ExecutionError_SYSTEM, phaseInfo.Err().Kind)
+	})
 }
 
 func TestDemystifyPending_testcases(t *testing.T) {