diff --git a/go/tasks/plugins/k8s/spark/spark.go b/go/tasks/plugins/k8s/spark/spark.go index f0d328b2c..18d11a507 100755 --- a/go/tasks/plugins/k8s/spark/spark.go +++ b/go/tasks/plugins/k8s/spark/spark.go @@ -413,8 +413,13 @@ func getEventInfoForSpark(pluginContext k8s.PluginContext, sj *sparkOp.SparkAppl }) } } else if sj.Status.AppState.State == sparkOp.RunningState && sj.Status.DriverInfo.WebUIIngressAddress != "" { - // Append https as the operator doesn't currently. - customInfoMap[sparkDriverUI] = fmt.Sprintf("https://%s", sj.Status.DriverInfo.WebUIIngressAddress) + // Older versions of spark-operator does not append http:// but newer versions do. + uri := sj.Status.DriverInfo.WebUIIngressAddress + if !strings.HasPrefix(uri, "https://") && !strings.HasPrefix(uri, "http://") { + uri = fmt.Sprintf("https://%s", uri) + } + customInfoMap[sparkDriverUI] = uri + // Custom doesn't work unless the UI has a custom plugin to parse this, hence add to Logs as well. taskLogs = append(taskLogs, &core.TaskLog{ Uri: customInfoMap[sparkDriverUI], diff --git a/go/tasks/plugins/k8s/spark/spark_test.go b/go/tasks/plugins/k8s/spark/spark_test.go index 5750643a2..efa52274b 100755 --- a/go/tasks/plugins/k8s/spark/spark_test.go +++ b/go/tasks/plugins/k8s/spark/spark_test.go @@ -2,7 +2,6 @@ package spark import ( "context" - "fmt" "os" "strconv" "testing" @@ -34,7 +33,7 @@ import ( const sparkMainClass = "MainClass" const sparkApplicationFile = "local:///spark_app.py" const testImage = "image://" -const sparkUIAddress = "spark-ui.flyte" +const sparkUIAddress = "https://spark-ui.flyte" var ( dummySparkConf = map[string]string{ @@ -92,7 +91,7 @@ func TestGetEventInfo(t *testing.T) { info, err := getEventInfoForSpark(taskCtx, dummySparkApplication(sj.RunningState)) assert.NoError(t, err) assert.Len(t, info.Logs, 6) - assert.Equal(t, fmt.Sprintf("https://%s", sparkUIAddress), info.CustomInfo.Fields[sparkDriverUI].GetStringValue()) + assert.Equal(t, "https://spark-ui.flyte", info.CustomInfo.Fields[sparkDriverUI].GetStringValue()) generatedLinks := make([]string, 0, len(info.Logs)) for _, l := range info.Logs { generatedLinks = append(generatedLinks, l.Uri)