Fix double http in the Spark Driver UI Link

go/tasks/plugins/k8s/spark/spark.go

@@ -413,8 +413,13 @@
 			})
 		}
 	} else if sj.Status.AppState.State == sparkOp.RunningState && sj.Status.DriverInfo.WebUIIngressAddress != "" {
-		// Append https as the operator doesn't currently.
-		customInfoMap[sparkDriverUI] = fmt.Sprintf("https://%s", sj.Status.DriverInfo.WebUIIngressAddress)
+		// Older versions of spark-operator does not append http:// but newer versions do.
+		uri := sj.Status.DriverInfo.WebUIIngressAddress
+		if !strings.HasPrefix(uri, "https://") && !strings.HasPrefix(uri, "http://") {
+			uri = fmt.Sprintf("https://%s", uri)
+		}
+		customInfoMap[sparkDriverUI] = uri
+
 		// Custom doesn't work unless the UI has a custom plugin to parse this, hence add to Logs as well.
 		taskLogs = append(taskLogs, &core.TaskLog{
 			Uri:           customInfoMap[sparkDriverUI],

go/tasks/plugins/k8s/spark/spark_test.go

@@ -2,7 +2,6 @@ package spark
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"strconv"
 	"testing"
@@ -34,7 +33,7 @@ import (
 const sparkMainClass = "MainClass"
 const sparkApplicationFile = "local:///spark_app.py"
 const testImage = "image://"
-const sparkUIAddress = "spark-ui.flyte"
+const sparkUIAddress = "https://spark-ui.flyte"
 
 var (
 	dummySparkConf = map[string]string{
@@ -92,7 +91,7 @@ func TestGetEventInfo(t *testing.T) {
 	info, err := getEventInfoForSpark(taskCtx, dummySparkApplication(sj.RunningState))
 	assert.NoError(t, err)
 	assert.Len(t, info.Logs, 6)
-	assert.Equal(t, fmt.Sprintf("https://%s", sparkUIAddress), info.CustomInfo.Fields[sparkDriverUI].GetStringValue())
+	assert.Equal(t, "https://spark-ui.flyte", info.CustomInfo.Fields[sparkDriverUI].GetStringValue())
 	generatedLinks := make([]string, 0, len(info.Logs))
 	for _, l := range info.Logs {
 		generatedLinks = append(generatedLinks, l.Uri)