Skip to content

Commit

Permalink
Fix integration tests (#5417)
Browse files Browse the repository at this point in the history
  • Loading branch information
@RustedBones
RustedBones authored Jul 4, 2024
1 parent 4e7f918 commit a5f580d
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 24 deletions.
24 changes: 18 additions & 6 deletions integration/src/test/scala/com/spotify/scio/bigquery/types/BigQueryStorageIT.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ class BigQueryStorageIT extends AnyFlatSpec with Matchers {
t.plus(Duration.millis(i.toLong)),
dt.toLocalDate.plusDays(i),
dt.toLocalTime.plusMillis(i),
dt.toLocalDateTime.plusMillis(i)
dt.toLocalDateTime.plusMillis(i),
s"POINT($i $i)", // geography is not an avro logical type
s"""{"value":$i}""" // json is not an avro logical type
)
}.asJava
val (sc, _) = ContextAndArgs(
Expand All @@ -74,7 +76,9 @@ class BigQueryStorageIT extends AnyFlatSpec with Matchers {
Some(t.plus(Duration.millis(i.toLong))),
Some(dt.toLocalDate.plusDays(i)),
Some(dt.toLocalTime.plusMillis(i)),
Some(dt.toLocalDateTime.plusMillis(i))
Some(dt.toLocalDateTime.plusMillis(i)),
Some(s"POINT($i $i)"), // geography is not an avro logical type
Some(s"""{"value":$i}""") // json is not an avro logical type
)
}.asJava
val (sc, _) = ContextAndArgs(
Expand All @@ -99,7 +103,9 @@ class BigQueryStorageIT extends AnyFlatSpec with Matchers {
List(t.plus(Duration.millis(i.toLong))),
List(dt.toLocalDate.plusDays(i)),
List(dt.toLocalTime.plusMillis(i)),
List(dt.toLocalDateTime.plusMillis(i))
List(dt.toLocalDateTime.plusMillis(i)),
List(s"POINT($i $i)"), // geography is not an avro logical type
List(s"""{"value":$i}""") // json is not an avro logical type
)
}.asJava
val (sc, _) = ContextAndArgs(
Expand Down Expand Up @@ -184,7 +190,9 @@ class BigQueryStorageIT extends AnyFlatSpec with Matchers {
Some(t.plus(Duration.millis(i.toLong))),
Some(dt.toLocalDate.plusDays(i)),
Some(dt.toLocalTime.plusMillis(i)),
Some(dt.toLocalDateTime.plusMillis(i))
Some(dt.toLocalDateTime.plusMillis(i)),
Some(Geography(s"POINT($i $i)")),
Some(Json(s"""{"value":$i}"""))
)
}.asJava
val (sc, _) = ContextAndArgs(
Expand All @@ -211,7 +219,9 @@ class BigQueryStorageIT extends AnyFlatSpec with Matchers {
t.plus(Duration.millis(i.toLong)),
dt.toLocalDate.plusDays(i),
dt.toLocalTime.plusMillis(i),
dt.toLocalDateTime.plusMillis(i)
dt.toLocalDateTime.plusMillis(i),
Geography(s"POINT($i $i)"),
Json(s"""{"value":$i}""")
)
}.asJava
val (sc, _) = ContextAndArgs(
Expand Down Expand Up @@ -248,7 +258,9 @@ class BigQueryStorageIT extends AnyFlatSpec with Matchers {
Some(t.plus(Duration.millis(i.toLong))),
Some(dt.toLocalDate.plusDays(i)),
Some(dt.toLocalTime.plusMillis(i)),
Some(dt.toLocalDateTime.plusMillis(i))
Some(dt.toLocalDateTime.plusMillis(i)),
Some(Geography(s"POINT($i $i)")),
Some(Json(s"""{"value":$i}"""))
)
}.asJava
val (sc, _) = ContextAndArgs(
Expand Down
36 changes: 18 additions & 18 deletions scio-google-cloud-platform/src/main/scala/com/spotify/scio/bigquery/types/TypeProvider.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,31 +262,31 @@ private[types] object TypeProvider {
val provider: OverrideTypeProvider = OverrideTypeProviderFinder.getProvider

// Returns: (raw type, e.g. Int, String, NestedRecord, nested case class definitions)
def getRawType(tfs: TableFieldSchema): (Tree, Seq[Tree]) =
def getRawType(tfs: TableFieldSchema): (Tree, Seq[Tree]) = {
// format: off
tfs.getType match {
case _ if provider.shouldOverrideType(tfs) =>
(provider.getScalaType(c)(tfs), Nil)
case "BOOLEAN" | "BOOL" => (tq"_root_.scala.Boolean", Nil)
case "INTEGER" | "INT64" => (tq"_root_.scala.Long", Nil)
case "FLOAT" | "FLOAT64" => (tq"_root_.scala.Double", Nil)
case "STRING" => (tq"_root_.java.lang.String", Nil)
case "NUMERIC" => (tq"_root_.scala.BigDecimal", Nil)
case "BIGNUMERIC" => (tq"_root_.scala.BigDecimal", Nil)
case "BYTES" => (tq"_root_.com.google.protobuf.ByteString", Nil)
case "TIMESTAMP" => (tq"_root_.org.joda.time.Instant", Nil)
case "DATE" => (tq"_root_.org.joda.time.LocalDate", Nil)
case "TIME" => (tq"_root_.org.joda.time.LocalTime", Nil)
case "DATETIME" => (tq"_root_.org.joda.time.LocalDateTime", Nil)
case "GEOGRAPHY" =>
(tq"_root_.com.spotify.scio.bigquery.types.Geography", Nil)
case "JSON" =>
(tq"_root_.com.spotify.scio.bigquery.types.Json", Nil)
case _ if provider.shouldOverrideType(tfs) => (provider.getScalaType(c)(tfs), Nil)
case "BOOLEAN" | "BOOL" => (tq"_root_.scala.Boolean", Nil)
case "INTEGER" | "INT64" => (tq"_root_.scala.Long", Nil)
case "FLOAT" | "FLOAT64" => (tq"_root_.scala.Double", Nil)
case "STRING" => (tq"_root_.java.lang.String", Nil)
case "NUMERIC" => (tq"_root_.scala.BigDecimal", Nil)
case "BIGNUMERIC" => (tq"_root_.scala.BigDecimal", Nil)
case "BYTES" => (tq"_root_.com.google.protobuf.ByteString", Nil)
case "TIMESTAMP" => (tq"_root_.org.joda.time.Instant", Nil)
case "DATE" => (tq"_root_.org.joda.time.LocalDate", Nil)
case "TIME" => (tq"_root_.org.joda.time.LocalTime", Nil)
case "DATETIME" => (tq"_root_.org.joda.time.LocalDateTime", Nil)
case "GEOGRAPHY" => (tq"_root_.com.spotify.scio.bigquery.types.Geography", Nil)
case "JSON" => (tq"_root_.com.spotify.scio.bigquery.types.Json", Nil)
case "RECORD" | "STRUCT" =>
val name = NameProvider.getUniqueName(tfs.getName)
val (fields, records) = toFields(tfs.getFields)
(q"${Ident(TypeName(name))}", Seq(q"case class ${TypeName(name)}(..$fields)") ++ records)
case t => c.abort(c.enclosingPosition, s"type: $t not supported")
}
// format: on
}

// Returns: (field type, e.g. T/Option[T]/List[T], nested case class definitions)
def getFieldType(tfs: TableFieldSchema): (Tree, Seq[Tree]) = {
Expand Down

0 comments on commit a5f580d

Please sign in to comment.