From 1e6fa651b5abb5f7a7de9c52875a91e5e3a10ec1 Mon Sep 17 00:00:00 2001 From: Andres Gomez Ferrer Date: Thu, 23 Nov 2023 16:47:37 +0100 Subject: [PATCH 1/3] Fix nested list & map of structs Signed-off-by: Andres Gomez Ferrer --- .../flytekitscala/SdkScalaTypeTest.scala | 72 +++++++++++++++++-- .../flyte/flytekitscala/SdkLiteralTypes.scala | 14 ++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/flytekit-scala-tests/src/test/scala/org/flyte/flytekitscala/SdkScalaTypeTest.scala b/flytekit-scala-tests/src/test/scala/org/flyte/flytekitscala/SdkScalaTypeTest.scala index 2424c823..93014c55 100644 --- a/flytekit-scala-tests/src/test/scala/org/flyte/flytekitscala/SdkScalaTypeTest.scala +++ b/flytekit-scala-tests/src/test/scala/org/flyte/flytekitscala/SdkScalaTypeTest.scala @@ -52,7 +52,9 @@ import org.flyte.flytekitscala.SdkLiteralTypes.{ case class ScalarNested( foo: String, bar: Option[String], - nestedNested: Option[ScalarNestedNested] + nestedNested: Option[ScalarNestedNested], + nestedNestedList: List[ScalarNestedNested], + nestedNestedMap: Map[String, ScalarNestedNested] ) case class ScalarNestedNested(foo: String, bar: Option[String]) @@ -191,6 +193,32 @@ class SdkScalaTypeTest { "bar" -> Struct.Value.ofStringValue("bar") ).asJava ) + ), + "nestedNestedList" -> Struct.Value.ofListValue( + List( + Struct.Value.ofStructValue( + Struct.of( + Map( + "foo" -> Struct.Value.ofStringValue("foo"), + "bar" -> Struct.Value.ofStringValue("bar") + ).asJava + ) + ) + ).asJava + ), + "nestedNestedMap" -> Struct.Value.ofStructValue( + Struct.of( + Map( + "foo" -> Struct.Value.ofStructValue( + Struct.of( + Map( + "foo" -> Struct.Value.ofStringValue("foo"), + "bar" -> Struct.Value.ofStringValue("bar") + ).asJava + ) + ) + ).asJava + ) ) ).asJava ) @@ -212,7 +240,9 @@ class SdkScalaTypeTest { ScalarNested( "foo", None, - Some(ScalarNestedNested("foo", Some("bar"))) + Some(ScalarNestedNested("foo", Some("bar"))), + List(ScalarNestedNested("foo", Some("bar"))), + Map("foo" -> ScalarNestedNested("foo", Some("bar"))) ) ) ) @@ -238,7 +268,9 @@ class SdkScalaTypeTest { ScalarNested( "foo", Some("bar"), - Some(ScalarNestedNested("foo", Some("bar"))) + Some(ScalarNestedNested("foo", Some("bar"))), + List(ScalarNestedNested("foo", Some("bar"))), + Map("foo" -> ScalarNestedNested("foo", Some("bar"))) ) ) ) @@ -274,6 +306,32 @@ class SdkScalaTypeTest { "bar" -> Struct.Value.ofStringValue("bar") ).asJava ) + ), + "nestedNestedList" -> Struct.Value.ofListValue( + List( + Struct.Value.ofStructValue( + Struct.of( + Map( + "foo" -> Struct.Value.ofStringValue("foo"), + "bar" -> Struct.Value.ofStringValue("bar") + ).asJava + ) + ) + ).asJava + ), + "nestedNestedMap" -> Struct.Value.ofStructValue( + Struct.of( + Map( + "foo" -> Struct.Value.ofStructValue( + Struct.of( + Map( + "foo" -> Struct.Value.ofStringValue("foo"), + "bar" -> Struct.Value.ofStringValue("bar") + ).asJava + ) + ) + ).asJava + ) ) ).asJava ) @@ -317,7 +375,9 @@ class SdkScalaTypeTest { ScalarNested( "foo", Some("bar"), - Some(ScalarNestedNested("foo", Some("bar"))) + Some(ScalarNestedNested("foo", Some("bar"))), + List(ScalarNestedNested("foo", Some("bar"))), + Map("foo" -> ScalarNestedNested("foo", Some("bar"))) ) ) ) @@ -337,7 +397,9 @@ class SdkScalaTypeTest { ScalarNested( "foo", Some("bar"), - Some(ScalarNestedNested("foo", Some("bar"))) + Some(ScalarNestedNested("foo", Some("bar"))), + List(ScalarNestedNested("foo", Some("bar"))), + Map("foo" -> ScalarNestedNested("foo", Some("bar"))) ) ) ).asJava diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala index 517ec24d..8acb1d16 100644 --- a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala +++ b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala @@ -308,6 +308,20 @@ object SdkLiteralTypes { value.asInstanceOf[Double].toLong } else if (tpe =:= typeOf[Float]) { value.asInstanceOf[Double].toFloat + } else if (tpe <:< typeOf[List[Any]]) { + value + .asInstanceOf[List[Any]] + .map(value => { + valueToParamValue(value, tpe.typeArgs.head) + }) + } else if (tpe <:< typeOf[Map[String, Any]]) { + value + .asInstanceOf[Map[String, Any]] + .view + .mapValues(value => { + valueToParamValue(value, tpe.typeArgs(1)) + }) + .toMap } else if (tpe <:< typeOf[Option[Any]]) { // this has to be before Product check because Option is a Product if (value == None) { // None is used to represent Struct.Value.Kind.NULL_VALUE when converting struct to map None From 38edfc54cf09f54dac250b65c430ccdeda55c3fe Mon Sep 17 00:00:00 2001 From: Andres Gomez Ferrer Date: Thu, 23 Nov 2023 16:52:15 +0100 Subject: [PATCH 2/3] Rename value to v in lambdas Signed-off-by: Andres Gomez Ferrer --- .../scala/org/flyte/flytekitscala/SdkLiteralTypes.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala index 8acb1d16..b9293359 100644 --- a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala +++ b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala @@ -311,15 +311,15 @@ object SdkLiteralTypes { } else if (tpe <:< typeOf[List[Any]]) { value .asInstanceOf[List[Any]] - .map(value => { - valueToParamValue(value, tpe.typeArgs.head) + .map(v => { + valueToParamValue(v, tpe.typeArgs.head) }) } else if (tpe <:< typeOf[Map[String, Any]]) { value .asInstanceOf[Map[String, Any]] .view - .mapValues(value => { - valueToParamValue(value, tpe.typeArgs(1)) + .mapValues(v => { + valueToParamValue(v, tpe.typeArgs(1)) }) .toMap } else if (tpe <:< typeOf[Option[Any]]) { // this has to be before Product check because Option is a Product From 96b61a1dea47157fe0071a6005d382e8ff40520e Mon Sep 17 00:00:00 2001 From: Andres Gomez Ferrer Date: Thu, 23 Nov 2023 17:13:25 +0100 Subject: [PATCH 3/3] Make it compatible with scala 2.12 Signed-off-by: Andres Gomez Ferrer --- .../src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala index b9293359..cefc3bbb 100644 --- a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala +++ b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala @@ -317,7 +317,6 @@ object SdkLiteralTypes { } else if (tpe <:< typeOf[Map[String, Any]]) { value .asInstanceOf[Map[String, Any]] - .view .mapValues(v => { valueToParamValue(v, tpe.typeArgs(1)) })