Merge branch 'master' into feature/error_handling_athena_dynamodb

Trianz-Akshay · Sep 30, 2024 · 4072aec · 4072aec
2 parents ef8f26f + bbe98f6
commit 4072aec
Show file tree

Hide file tree

Showing 12 changed files with 34 additions and 18 deletions.
diff --git a/.github/workflows/javadoc_sync.yaml b/.github/workflows/javadoc_sync.yaml
@@ -28,7 +28,7 @@ jobs:
         run: rm -rf com
 
       - name: Copy new files
-        run: cp -r target/site/apidocs/* .
+        run: cp -r target/reports/apidocs/* .
 
       - name: Commit files
         run: |

diff --git a/athena-dynamodb/pom.xml b/athena-dynamodb/pom.xml
@@ -13,7 +13,7 @@
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>bom</artifactId>
-                <version>2.28.1</version>
+                <version>2.28.11</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>

diff --git a/athena-elasticsearch/pom.xml b/athena-elasticsearch/pom.xml
@@ -127,7 +127,7 @@
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-client</artifactId>
-            <version>8.15.1</version>
+            <version>8.15.2</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/org.elasticsearch.client/elasticsearch-rest-high-level-client -->
         <dependency>

diff --git a/athena-gcs/pom.xml b/athena-gcs/pom.xml
@@ -75,7 +75,7 @@
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-storage</artifactId>
-            <version>2.42.0</version>
+            <version>2.43.1</version>
         </dependency>
         <!-- Test Dependencies -->
         <dependency>

diff --git a/athena-google-bigquery/pom.xml b/athena-google-bigquery/pom.xml
@@ -80,7 +80,7 @@
         <dependency>
             <groupId>nl.jqno.equalsverifier</groupId>
             <artifactId>equalsverifier</artifactId>
-            <version>3.16.2</version>
+            <version>3.17</version>
             <scope>test</scope>
         </dependency>
         <dependency>

diff --git a/...na-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcRecordHandler.java b/...na-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcRecordHandler.java
@@ -82,11 +82,11 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
 /**
  * Abstracts JDBC record handler and provides common reusable split records handling.
@@ -273,8 +273,9 @@ protected Extractor makeExtractor(Field field, ResultSet resultSet, Map<String,
             case DATEDAY:
                 return (DateDayExtractor) (Object context, NullableDateDayHolder dst) ->
                 {
+                    //Issue fix for getting different date (offset by 1) for any dates prior to 1/1/1970.
                     if (resultSet.getDate(fieldName) != null) {
-                        dst.value = (int) TimeUnit.MILLISECONDS.toDays(resultSet.getDate(fieldName).getTime());
+                        dst.value = (int) LocalDate.parse(resultSet.getDate(fieldName).toString()).toEpochDay();
                     }
                     dst.isSet = resultSet.wasNull() ? 0 : 1;
                 };

diff --git a/athena-kafka/pom.xml b/athena-kafka/pom.xml
@@ -24,7 +24,7 @@
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>
-            <version>3.8.0</version>
+            <version>7.7.1-ce</version>
         </dependency>
         <dependency>
             <groupId>org.apache.avro</groupId>

diff --git a/athena-msk/pom.xml b/athena-msk/pom.xml
@@ -81,7 +81,7 @@
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>
-            <version>3.8.0</version>
+            <version>7.7.1-ce</version>
         </dependency>
         <dependency>
             <groupId>org.apache.avro</groupId>
@@ -91,7 +91,7 @@
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>
-            <version>3.25.3</version>
+            <version>3.25.5</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.glue</groupId>

diff --git a/...-oracle/src/test/java/com/amazonaws/athena/connectors/oracle/OracleRecordHandlerTest.java b/...-oracle/src/test/java/com/amazonaws/athena/connectors/oracle/OracleRecordHandlerTest.java
@@ -45,8 +45,10 @@
 import org.mockito.Mockito;
 
 import java.sql.Connection;
+import java.sql.Date;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.time.LocalDate;
 import java.util.Collections;
 
 import static com.amazonaws.athena.connectors.oracle.OracleConstants.ORACLE_NAME;
@@ -97,6 +99,8 @@ public void buildSplitSql()
         schemaBuilder.addField(FieldBuilder.newBuilder("testCol6", Types.MinorType.TINYINT.getType()).build());
         schemaBuilder.addField(FieldBuilder.newBuilder("testCol7", Types.MinorType.FLOAT8.getType()).build());
         schemaBuilder.addField(FieldBuilder.newBuilder("testCol8", Types.MinorType.BIT.getType()).build());
+        schemaBuilder.addField(FieldBuilder.newBuilder("testCol9", Types.MinorType.DATEDAY.getType()).build());
+        schemaBuilder.addField(FieldBuilder.newBuilder("testCol10", Types.MinorType.DATEDAY.getType()).build());
         schemaBuilder.addField(FieldBuilder.newBuilder("partition_name", Types.MinorType.VARCHAR.getType()).build());
         Schema schema = schemaBuilder.build();
 
@@ -120,6 +124,10 @@ public void buildSplitSql()
         ValueSet valueSet6 = getSingleValueSet(0);
         ValueSet valueSet7 = getSingleValueSet(1.2d);
         ValueSet valueSet8 = getSingleValueSet(true);
+        final long epochDaysPrior1970 = LocalDate.parse("1967-07-27").toEpochDay();
+        ValueSet valueSet9 = getSingleValueSet(epochDaysPrior1970);
+        final long epochDaysPost1970 = LocalDate.parse("1971-01-01").toEpochDay();
+        ValueSet valueSet10 = getSingleValueSet(epochDaysPost1970);
 
         Constraints constraints = Mockito.mock(Constraints.class);
         Mockito.when(constraints.getSummary()).thenReturn(new ImmutableMap.Builder<String, ValueSet>()
@@ -131,11 +139,13 @@ public void buildSplitSql()
                 .put("testCol6", valueSet6)
                 .put("testCol7", valueSet7)
                 .put("testCol8", valueSet8)
+                .put("testCol9", valueSet9)
+                .put("testCol10", valueSet10)
                 .build());
 
         Mockito.when(constraints.getLimit()).thenReturn(5L);
 
-        String expectedSql = "SELECT \"testCol1\", \"testCol2\", \"testCol3\", \"testCol4\", \"testCol5\", \"testCol6\", \"testCol7\", \"testCol8\" FROM \"testSchema\".\"testTable\" PARTITION (p0)  WHERE (\"testCol1\" IN (?,?)) AND ((\"testCol2\" >= ? AND \"testCol2\" < ?)) AND ((\"testCol3\" > ? AND \"testCol3\" <= ?)) AND (\"testCol4\" = ?) AND (\"testCol5\" = ?) AND (\"testCol6\" = ?) AND (\"testCol7\" = ?) AND (\"testCol8\" = ?)";
+        String expectedSql = "SELECT \"testCol1\", \"testCol2\", \"testCol3\", \"testCol4\", \"testCol5\", \"testCol6\", \"testCol7\", \"testCol8\", \"testCol9\", \"testCol10\" FROM \"testSchema\".\"testTable\" PARTITION (p0)  WHERE (\"testCol1\" IN (?,?)) AND ((\"testCol2\" >= ? AND \"testCol2\" < ?)) AND ((\"testCol3\" > ? AND \"testCol3\" <= ?)) AND (\"testCol4\" = ?) AND (\"testCol5\" = ?) AND (\"testCol6\" = ?) AND (\"testCol7\" = ?) AND (\"testCol8\" = ?) AND (\"testCol9\" = ?) AND (\"testCol10\" = ?)";
         PreparedStatement expectedPreparedStatement = Mockito.mock(PreparedStatement.class);
         Mockito.when(this.connection.prepareStatement(Mockito.eq(expectedSql))).thenReturn(expectedPreparedStatement);
         PreparedStatement preparedStatement = this.oracleRecordHandler.buildSplitSql(this.connection, "testCatalogName", tableName, schema, constraints, split);
@@ -152,6 +162,11 @@ public void buildSplitSql()
         Mockito.verify(preparedStatement, Mockito.times(1)).setByte(9, (byte) 0);
         Mockito.verify(preparedStatement, Mockito.times(1)).setDouble(10, 1.2d);
         Mockito.verify(preparedStatement, Mockito.times(1)).setBoolean(11, true);
+        //year – the year minus 1900; must be 0 to 8099. (Note that 8099 is 9999 minus 1900.) month – 0 to 11 day – 1 to 31
+        Date expectedDatePrior1970 = new Date(67, 6, 27);//Date: 1967-07-27
+        Mockito.verify(preparedStatement, Mockito.times(1)).setDate(12, expectedDatePrior1970);
+        Date expectedDatePost1970 = new Date(71, 0, 1);//Date: 1971-01-01
+        Mockito.verify(preparedStatement, Mockito.times(1)).setDate(13, expectedDatePost1970);
     }
 
     private ValueSet getSingleValueSet(Object value) {

diff --git a/athena-saphana/pom.xml b/athena-saphana/pom.xml
@@ -62,7 +62,7 @@
             <groupId>com.sap.cloud.db.jdbc</groupId>
             <artifactId>ngdbc</artifactId>
             <!--            <version>2.11.17</version>-->
-            <version>2.21.11</version>
+            <version>2.22.11</version>
         </dependency>
     </dependencies>
     <build>

diff --git a/athena-synapse/pom.xml b/athena-synapse/pom.xml
@@ -35,7 +35,7 @@
         <dependency>
             <groupId>com.microsoft.azure</groupId>
             <artifactId>msal4j</artifactId>
-            <version>1.17.1</version>
+            <version>1.17.2</version>
             <exclusions>
                 <exclusion>
                     <groupId>com.fasterxml.jackson.datatype</groupId>

diff --git a/pom.xml b/pom.xml
@@ -27,11 +27,11 @@
         <assertj.version>3.26.3</assertj.version>
         <testng.version>7.10.2</testng.version>
          <!--- to meet engine version 2.12.6 -->
-        <fasterxml.jackson.version>2.17.2</fasterxml.jackson.version>
+        <fasterxml.jackson.version>2.18.0</fasterxml.jackson.version>
         <surefire.failsafe.version>3.5.0</surefire.failsafe.version>
-        <log4j2Version>2.24.0</log4j2Version>
+        <log4j2Version>2.24.1</log4j2Version>
         <apache.arrow.version>13.0.0</apache.arrow.version>
-        <guava.version>33.3.0-jre</guava.version>
+        <guava.version>33.3.1-jre</guava.version>
         <protobuf3.version>3.25.3</protobuf3.version>
         <antlr.st4.version>4.3.4</antlr.st4.version>
         <log4j2.cachefile.transformer.version>2.15</log4j2.cachefile.transformer.version>
@@ -53,7 +53,7 @@
             <dependency>
                 <groupId>org.junit</groupId>
                 <artifactId>junit-bom</artifactId>
-                <version>5.11.0</version>
+                <version>5.11.1</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -432,7 +432,7 @@
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-gpg-plugin</artifactId>
-                        <version>3.2.6</version>
+                        <version>3.2.7</version>
                         <executions>
                             <execution>
                                 <id>sign-artifacts</id>