From 0d6bdb1144843952de35da94269f6600345bdc55 Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Tue, 23 Jun 2020 23:35:17 +0530 Subject: [PATCH] Support ADLS gen1/2 connectivity via Hadoop 3.2 jars (#560) 1) Hadoop 3.2.0 support 2) Increase the max width of CHAR to 255. (#556). Fix provided by @neeraj-tibco 3) Return table type as 'table' when property 'metadatacase.lower' is set to true. (#557). Fix for SDENT-194 Co-authored-by: Sumedh Wale Co-authored-by: Nikhil Bandiwadekar Co-authored-by: Trilok Khairnar Co-authored-by: Amogh Shetkar --- build.gradle | 4 ++-- gemfire-core/build.gradle | 1 + .../hdfs/internal/org/apache/hadoop/io/SequenceFile.java | 2 +- gemfire-junit/build.gradle | 1 + .../gemfirexd/internal/engine/store/RowFormatter.java | 4 ++-- .../internal/impl/jdbc/EmbedDatabaseMetaData.java | 2 +- .../tools/planexporter/AccessDistributedSystem.java | 2 +- .../gemfirexd/internal/impl/jdbc/metadata_lc.properties | 7 ++++--- .../gemfirexd/internal/shared/common/reference/Limits.java | 2 +- 9 files changed, 14 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 5edcf84f7..85063c25f 100644 --- a/build.gradle +++ b/build.gradle @@ -59,7 +59,7 @@ allprojects { pxfVersion = '2.5.1.0' osgiVersion = '6.0.0' jettyVersion = '9.2.26.v20180806' - hadoopVersion = '2.7.7' + hadoopVersion = '3.2.0' protobufVersion = '2.6.1' kryoVersion = '4.0.2' thriftVersion = '0.9.3' @@ -131,7 +131,7 @@ allprojects { PRODUCT_MAJOR = '1' PRODUCT_MINOR = '6' PRODUCT_MAINT = '5' - PRODUCT_CLASSIFIER = '' + PRODUCT_CLASSIFIER = '-HF-1' PRODUCT_RELEASE_STAGE = '' PRODUCT_VERSION = "${PRODUCT_MAJOR}.${PRODUCT_MINOR}.${PRODUCT_MAINT}${PRODUCT_CLASSIFIER}" vendorName = 'TIBCO Software Inc.' diff --git a/gemfire-core/build.gradle b/gemfire-core/build.gradle index f5fd41dff..668d10803 100644 --- a/gemfire-core/build.gradle +++ b/gemfire-core/build.gradle @@ -69,6 +69,7 @@ dependencies { compileOnly("org.apache.hadoop:hadoop-auth:${hadoopVersion}") compileOnly("org.apache.hadoop:hadoop-common:${hadoopVersion}") compileOnly "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}" + compileOnly "org.apache.hadoop:hadoop-hdfs-client:${hadoopVersion}" compileOnly "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}" compileOnly "com.google.protobuf:protobuf-java:${protobufVersion}" compileOnly "org.glassfish.jersey.core:jersey-server:${jerseyVersion}" diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/cache/hdfs/internal/org/apache/hadoop/io/SequenceFile.java b/gemfire-core/src/main/java/com/gemstone/gemfire/cache/hdfs/internal/org/apache/hadoop/io/SequenceFile.java index b5f78bc26..aaf39bc53 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/cache/hdfs/internal/org/apache/hadoop/io/SequenceFile.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/cache/hdfs/internal/org/apache/hadoop/io/SequenceFile.java @@ -1274,7 +1274,7 @@ public void sync() throws IOException { @Deprecated public void syncFs() throws IOException { if (out != null) { - out.sync(); // flush contents to file system + out.hsync(); // flush contents to file system } } diff --git a/gemfire-junit/build.gradle b/gemfire-junit/build.gradle index f1738dba5..a80c4bdd6 100644 --- a/gemfire-junit/build.gradle +++ b/gemfire-junit/build.gradle @@ -55,6 +55,7 @@ dependencies { compile "org.apache.hadoop:hadoop-auth:${hadoopVersion}" compile "org.apache.hadoop:hadoop-common:${hadoopVersion}" compile "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}" + compile "org.apache.hadoop:hadoop-hdfs-client:${hadoopVersion}" compile "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}" compile "com.google.protobuf:protobuf-java:${protobufVersion}" compile "org.apache.hbase:hbase-common:${hbaseVersion}" diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java index 9d6c22e2b..f71942262 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java @@ -100,7 +100,7 @@ * @author Rahul Dubey * @author swale */ -@SuppressWarnings("serial") +@SuppressWarnings({ "serial", "WeakerAccess" }) public final class RowFormatter implements Serializable { /* @@ -179,7 +179,7 @@ public final class RowFormatter implements Serializable { /** * Number of bytes required for serializing {@link #TOKEN_RECOVERY_VERSION}. */ - static final int TOKEN_RECOVERY_VERSION_BYTES = getCompactIntNumBytes( + private static final int TOKEN_RECOVERY_VERSION_BYTES = getCompactIntNumBytes( TOKEN_RECOVERY_VERSION); /** diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java index a2765ed25..0a27c7377 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java @@ -165,7 +165,7 @@ private Properties getProperties(boolean net) { Properties p; if (net) { p = queryDescriptions_net; - } else if (System.getProperty(METADATACASE_LOWER_PROP) != null + } else if (System.getProperty(METADATACASE_LOWER_PROP, "false").equalsIgnoreCase("true") && getLanguageConnectionContext().isQueryRoutingFlagTrue()) { p = queryDescriptions_lc; } else { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/planexporter/AccessDistributedSystem.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/planexporter/AccessDistributedSystem.java index 7e044c0ca..8d15ed2c8 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/planexporter/AccessDistributedSystem.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/planexporter/AccessDistributedSystem.java @@ -538,7 +538,7 @@ private boolean schemaExists() throws SQLException { boolean found = false; ResultSet result = conn.getMetaData().getSchemas(); while (result.next()) { - boolean schemaMatches = System.getProperty(METADATACASE_LOWER_PROP) != null && lcc.isQueryRoutingFlagTrue() ? + boolean schemaMatches = System.getProperty(METADATACASE_LOWER_PROP, "false").equalsIgnoreCase("true") && lcc.isQueryRoutingFlagTrue() ? result.getString(1).equalsIgnoreCase(schema) : result.getString(1).equals(schema); if (schemaMatches) { found = true; diff --git a/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata_lc.properties b/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata_lc.properties index 897c5bc47..8177ce415 100644 --- a/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata_lc.properties +++ b/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata_lc.properties @@ -168,7 +168,8 @@ getTables=\ SELECT CAST ('' AS VARCHAR(128)) AS TABLE_CAT, \ LOWER (SCHEMANAME) AS TABLE_SCHEM, \ LOWER (TABLENAME) AS TABLE_NAME, \ - LOWER ((CAST (RTRIM(TABLE_TYPE) AS VARCHAR(16)))) \ + CASE WHEN LOWER ((CAST (RTRIM(TABLE_TYPE) AS VARCHAR(16)))) LIKE '%table%' THEN 'table' \ + ELSE LOWER ((CAST (RTRIM(TABLE_TYPE) AS VARCHAR(16)))) END \ AS TABLE_TYPE, LOWER (CAST ('' AS VARCHAR(128))) AS REMARKS, \ CAST (NULL AS VARCHAR(128)) AS TYPE_CAT, \ CAST (NULL AS VARCHAR(128)) AS TYPE_SCHEM, \ @@ -178,7 +179,7 @@ getTables=\ FROM \ SYS.SYSTABLES, \ SYS.SYSSCHEMAS, \ - (VALUES ('T','ROW TABLE'), ('S','SYSTEM TABLE'), \ + (VALUES ('T','TABLE'), ('S','SYSTEM TABLE'), \ ('V', 'VIEW'), ('A', 'SYNONYM')) T(TTABBREV,TABLE_TYPE) \ WHERE (TTABBREV=TABLETYPE \ AND (SYS.SYSTABLES.SCHEMAID = SYS.SYSSCHEMAS.SCHEMAID) \ @@ -193,7 +194,7 @@ UNION \ LOWER (SCHEMANAME) AS TABLE_SCHEM, \ LOWER (CAST (TABLENAME AS VARCHAR(128))) AS TABLE_NAME, \ LOWER (CASE WHEN TABLETYPE = 'VIEW' THEN 'VIEW' \ - ELSE CAST ((TABLETYPE || ' TABLE') AS VARCHAR(16)) END) AS TABLE_TYPE, \ + ELSE CAST (('TABLE') AS VARCHAR(16)) END) AS TABLE_TYPE, \ CAST ('' AS VARCHAR(128)) AS REMARKS, \ CAST (NULL AS VARCHAR(128)) AS TYPE_CAT, \ CAST (NULL AS VARCHAR(128)) AS TYPE_SCHEM, \ diff --git a/gemfirexd/shared/src/main/java/com/pivotal/gemfirexd/internal/shared/common/reference/Limits.java b/gemfirexd/shared/src/main/java/com/pivotal/gemfirexd/internal/shared/common/reference/Limits.java index a46c38afc..a03f7ac76 100644 --- a/gemfirexd/shared/src/main/java/com/pivotal/gemfirexd/internal/shared/common/reference/Limits.java +++ b/gemfirexd/shared/src/main/java/com/pivotal/gemfirexd/internal/shared/common/reference/Limits.java @@ -58,7 +58,7 @@ public interface Limits { * Trigger, Column, Schema, Savepoint, Table and View names) * are limited to 128 */ public static final int MAX_IDENTIFIER_LENGTH = 128; - public static final int DB2_CHAR_MAXWIDTH = 254; + public static final int DB2_CHAR_MAXWIDTH = 255; public static final int DB2_VARCHAR_MAXWIDTH = 32672; public static final int DB2_LOB_MAXWIDTH = 2147483647; public static final int DB2_LONGVARCHAR_MAXWIDTH = 32700;