From ad3f91b1d4e3a263dccea01c67172afb95bc4994 Mon Sep 17 00:00:00 2001 From: minurajeeve Date: Thu, 5 Oct 2023 15:03:16 +0530 Subject: [PATCH 1/3] Fix set-type directive for decimal by adding a default value for scale in case it is null/not provided. --- .../src/main/java/io/cdap/wrangler/utils/ColumnConverter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java b/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java index be9c7f0bf..777eb5048 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java @@ -317,7 +317,8 @@ public static Schema getSchemaForType(String type, Integer scale) throws Directi type = type.toUpperCase(); if (type.equals(ColumnTypeNames.DECIMAL)) { // TODO make set-type support setting decimal precision - typeSchema = Schema.nullableOf(Schema.decimalOf(38, scale)); + scale = scale != null ? scale : 38; + typeSchema = Schema.nullableOf(Schema.decimalOf(76, scale)); } else { if (!SCHEMA_TYPE_MAP.containsKey(type)) { throw new DirectiveParseException(String.format("'%s' is an unsupported type. " + From 540dba476294ceeae0f3d30b4befa79caac2b195 Mon Sep 17 00:00:00 2001 From: minurajeeve Date: Fri, 6 Oct 2023 18:11:29 +0530 Subject: [PATCH 2/3] Adding test case when scale is null --- .../java/io/cdap/wrangler/utils/ColumnConverter.java | 2 +- .../java/io/cdap/directives/column/SetTypeTest.java | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java b/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java index 777eb5048..29d64a596 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/utils/ColumnConverter.java @@ -318,7 +318,7 @@ public static Schema getSchemaForType(String type, Integer scale) throws Directi if (type.equals(ColumnTypeNames.DECIMAL)) { // TODO make set-type support setting decimal precision scale = scale != null ? scale : 38; - typeSchema = Schema.nullableOf(Schema.decimalOf(76, scale)); + typeSchema = Schema.nullableOf(Schema.decimalOf(77, scale)); } else { if (!SCHEMA_TYPE_MAP.containsKey(type)) { throw new DirectiveParseException(String.format("'%s' is an unsupported type. " + diff --git a/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java b/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java index 1d53727c4..cb617696c 100644 --- a/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java +++ b/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java @@ -213,6 +213,17 @@ public void testToDecimalInvalidRoundingMode() throws Exception { TestingRig.execute(directives, rows); } + @Test + public void testToDecimalScaleIsNull() throws Exception { + List rows = Collections.singletonList(new Row("scale_2", "125.45")); + String[] directives = new String[] {"set-type scale_2 decimal"}; + List results = TestingRig.execute(directives, rows); + Row row = results.get(0); + + Assert.assertTrue(row.getValue(0) instanceof BigDecimal); + Assert.assertEquals(row.getValue(0), new BigDecimal("125.45")); + } + @Test public void testToBoolean() throws Exception { List trueRows = Collections.singletonList( From ab025fe8e4cd0ae4f8d55214ccd0a3e6fc2c67e7 Mon Sep 17 00:00:00 2001 From: minurajeeve Date: Mon, 9 Oct 2023 11:54:32 +0530 Subject: [PATCH 3/3] Adding test case when scale is null --- .../cdap/directives/column/SetTypeTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java b/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java index cb617696c..b5f83c9e3 100644 --- a/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java +++ b/wrangler-core/src/test/java/io/cdap/directives/column/SetTypeTest.java @@ -217,11 +217,30 @@ public void testToDecimalInvalidRoundingMode() throws Exception { public void testToDecimalScaleIsNull() throws Exception { List rows = Collections.singletonList(new Row("scale_2", "125.45")); String[] directives = new String[] {"set-type scale_2 decimal"}; + Schema inputSchema = Schema.recordOf( + "inputSchema", + Schema.Field.of("scale_2", Schema.of(Schema.Type.DOUBLE)) + ); + + Schema expectedSchema = Schema.recordOf( + "expectedSchema", + Schema.Field.of("scale_2", Schema.decimalOf(77, 38)) + ); + List results = TestingRig.execute(directives, rows); + Schema outputSchema = TestingRig.executeAndGetSchema(directives, rows, inputSchema); Row row = results.get(0); + Schema.Field outputSchemaField = expectedSchema.getFields().get(0); Assert.assertTrue(row.getValue(0) instanceof BigDecimal); Assert.assertEquals(row.getValue(0), new BigDecimal("125.45")); + + Assert.assertEquals(outputSchemaField.getSchema().getType(), + outputSchema.getField(outputSchemaField.getName()).getSchema().getNonNullable().getType()); + Assert.assertEquals(outputSchemaField.getSchema().getPrecision(), + outputSchema.getField(outputSchemaField.getName()).getSchema().getNonNullable().getPrecision()); + Assert.assertEquals(outputSchemaField.getSchema().getScale(), + outputSchema.getField(outputSchemaField.getName()).getSchema().getNonNullable().getScale()); } @Test