Destination S3V2: Avro does not fail on unsupported string formats (#…

…50420)
airbytehq · Dec 24, 2024 · 589705c · 589705c
1 parent e4e4636
commit 589705c
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/...k/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/json/JsonSchemaToAirbyteType.kt b/...k/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/json/JsonSchemaToAirbyteType.kt
@@ -8,8 +8,11 @@ import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.databind.node.JsonNodeFactory
 import com.fasterxml.jackson.databind.node.ObjectNode
 import io.airbyte.cdk.load.data.*
+import io.github.oshai.kotlinlogging.KotlinLogging
 
 class JsonSchemaToAirbyteType {
+    private val log = KotlinLogging.logger {}
+
     fun convert(schema: JsonNode): AirbyteType = convertInner(schema)!!
 
     private fun convertInner(schema: JsonNode): AirbyteType? {
@@ -87,7 +90,10 @@ class JsonSchemaToAirbyteType {
                     TimestampTypeWithTimezone
                 }
             null -> StringType
-            else -> UnknownType(schema)
+            else -> {
+                log.warn { "Ignoring unrecognized string format: ${schema.get("format").asText()}" }
+                StringType
+            }
         }
 
     private fun fromNumber(schema: ObjectNode): AirbyteType =

diff --git a/...e/load/src/test/kotlin/io/airbyte/cdk/load/data/json/JsonSchemaToAirbyteSchemaTypeTest.kt b/...e/load/src/test/kotlin/io/airbyte/cdk/load/data/json/JsonSchemaToAirbyteSchemaTypeTest.kt
@@ -251,4 +251,11 @@ class JsonSchemaToAirbyteSchemaTypeTest {
         val airbyteType = JsonSchemaToAirbyteType().convert(inputSchema)
         Assertions.assertEquals(UnionType.of(StringType, IntegerType), airbyteType)
     }
+
+    @Test
+    fun testUnrecognizedStringFormats() {
+        val schemaNode = ofType("string").put("format", "foo")
+        val airbyteType = JsonSchemaToAirbyteType().convert(schemaNode)
+        Assertions.assertTrue(airbyteType is StringType)
+    }
 }