Support ADLS gen1/2 connectivity via Hadoop 3.2 jars (#560)

1) Hadoop 3.2.0 support 2) Increase the max width of CHAR to 255. (#556). Fix provided by @neeraj-tibco 3) Return table type as 'table' when property 'metadatacase.lower' is set to true. (#557). Fix for SDENT-194 Co-authored-by: Sumedh Wale <[email protected]> Co-authored-by: Nikhil Bandiwadekar <[email protected]> Co-authored-by: Trilok Khairnar <[email protected]> Co-authored-by: Amogh Shetkar <[email protected]>
TIBCOSoftware · Jun 23, 2020 · 0d6bdb1 · 0d6bdb1
1 parent aeae20a
commit 0d6bdb1
Show file tree

Hide file tree

Showing 9 changed files with 14 additions and 11 deletions.
diff --git a/build.gradle b/build.gradle
@@ -59,7 +59,7 @@ allprojects {
     pxfVersion = '2.5.1.0'
     osgiVersion = '6.0.0'
     jettyVersion = '9.2.26.v20180806'
-    hadoopVersion = '2.7.7'
+    hadoopVersion = '3.2.0'
     protobufVersion = '2.6.1'
     kryoVersion = '4.0.2'
     thriftVersion = '0.9.3'
@@ -131,7 +131,7 @@ allprojects {
     PRODUCT_MAJOR = '1'
     PRODUCT_MINOR = '6'
     PRODUCT_MAINT = '5'
-    PRODUCT_CLASSIFIER = ''
+    PRODUCT_CLASSIFIER = '-HF-1'
     PRODUCT_RELEASE_STAGE = ''
     PRODUCT_VERSION = "${PRODUCT_MAJOR}.${PRODUCT_MINOR}.${PRODUCT_MAINT}${PRODUCT_CLASSIFIER}"
     vendorName = 'TIBCO Software Inc.'

diff --git a/gemfire-core/build.gradle b/gemfire-core/build.gradle
@@ -69,6 +69,7 @@ dependencies {
   compileOnly("org.apache.hadoop:hadoop-auth:${hadoopVersion}")
   compileOnly("org.apache.hadoop:hadoop-common:${hadoopVersion}")
   compileOnly "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}"
+  compileOnly "org.apache.hadoop:hadoop-hdfs-client:${hadoopVersion}"
   compileOnly "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}"
   compileOnly "com.google.protobuf:protobuf-java:${protobufVersion}"
   compileOnly "org.glassfish.jersey.core:jersey-server:${jerseyVersion}"

diff --git a/...main/java/com/gemstone/gemfire/cache/hdfs/internal/org/apache/hadoop/io/SequenceFile.java b/...main/java/com/gemstone/gemfire/cache/hdfs/internal/org/apache/hadoop/io/SequenceFile.java
@@ -1274,7 +1274,7 @@ public void sync() throws IOException {
     @Deprecated
     public void syncFs() throws IOException {
       if (out != null) {
-        out.sync();                               // flush contents to file system
+        out.hsync();                               // flush contents to file system
       }
     }
 

diff --git a/gemfire-junit/build.gradle b/gemfire-junit/build.gradle
@@ -55,6 +55,7 @@ dependencies {
   compile "org.apache.hadoop:hadoop-auth:${hadoopVersion}"
   compile "org.apache.hadoop:hadoop-common:${hadoopVersion}"
   compile "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}"
+  compile "org.apache.hadoop:hadoop-hdfs-client:${hadoopVersion}"
   compile "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}"
   compile "com.google.protobuf:protobuf-java:${protobufVersion}"
   compile "org.apache.hbase:hbase-common:${hbaseVersion}"

diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java
@@ -100,7 +100,7 @@
  * @author Rahul Dubey
  * @author swale
  */
-@SuppressWarnings("serial")
+@SuppressWarnings({ "serial", "WeakerAccess" })
 public final class RowFormatter implements Serializable {
 
   /*
@@ -179,7 +179,7 @@ public final class RowFormatter implements Serializable {
   /**
    * Number of bytes required for serializing {@link #TOKEN_RECOVERY_VERSION}.
    */
-  static final int TOKEN_RECOVERY_VERSION_BYTES = getCompactIntNumBytes(
+  private static final int TOKEN_RECOVERY_VERSION_BYTES = getCompactIntNumBytes(
       TOKEN_RECOVERY_VERSION);
 
   /**

diff --git a/...xd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java b/...xd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java
@@ -165,7 +165,7 @@ private Properties getProperties(boolean net) {
         Properties p;
         if (net) {
             p = queryDescriptions_net;
-        } else if (System.getProperty(METADATACASE_LOWER_PROP) != null
+        } else if (System.getProperty(METADATACASE_LOWER_PROP, "false").equalsIgnoreCase("true")
             && getLanguageConnectionContext().isQueryRoutingFlagTrue()) {
             p = queryDescriptions_lc;
         } else {

diff --git a/.../core/src/main/java/com/pivotal/gemfirexd/tools/planexporter/AccessDistributedSystem.java b/.../core/src/main/java/com/pivotal/gemfirexd/tools/planexporter/AccessDistributedSystem.java
@@ -538,7 +538,7 @@ private boolean schemaExists() throws SQLException {
     boolean found = false;
     ResultSet result = conn.getMetaData().getSchemas();
     while (result.next()) {
-      boolean schemaMatches = System.getProperty(METADATACASE_LOWER_PROP) != null && lcc.isQueryRoutingFlagTrue() ?
+      boolean schemaMatches = System.getProperty(METADATACASE_LOWER_PROP, "false").equalsIgnoreCase("true") && lcc.isQueryRoutingFlagTrue() ?
           result.getString(1).equalsIgnoreCase(schema) : result.getString(1).equals(schema);
       if (schemaMatches) {
         found = true;

diff --git a/...ebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata_lc.properties b/...ebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata_lc.properties
@@ -168,7 +168,8 @@ getTables=\
 	SELECT CAST ('' AS VARCHAR(128)) AS TABLE_CAT, \
 		LOWER (SCHEMANAME) AS TABLE_SCHEM,  \
 		LOWER (TABLENAME) AS TABLE_NAME, \
-		LOWER ((CAST (RTRIM(TABLE_TYPE) AS VARCHAR(16)))) \
+		CASE WHEN LOWER ((CAST (RTRIM(TABLE_TYPE) AS VARCHAR(16)))) LIKE '%table%' THEN 'table' \
+			ELSE LOWER ((CAST (RTRIM(TABLE_TYPE) AS VARCHAR(16)))) END \
 		    AS TABLE_TYPE, LOWER (CAST ('' AS VARCHAR(128))) AS REMARKS, \
                 CAST (NULL AS VARCHAR(128)) AS TYPE_CAT, \
                 CAST (NULL AS VARCHAR(128)) AS TYPE_SCHEM, \
@@ -178,7 +179,7 @@ getTables=\
 	FROM \
 		 SYS.SYSTABLES, \
 	     SYS.SYSSCHEMAS, \
-	     (VALUES ('T','ROW TABLE'), ('S','SYSTEM TABLE'), \
+	     (VALUES ('T','TABLE'), ('S','SYSTEM TABLE'), \
 			('V', 'VIEW'), ('A', 'SYNONYM')) T(TTABBREV,TABLE_TYPE) \
 	WHERE (TTABBREV=TABLETYPE 	\
 	  AND (SYS.SYSTABLES.SCHEMAID = SYS.SYSSCHEMAS.SCHEMAID) \
@@ -193,7 +194,7 @@ UNION \
 		LOWER (SCHEMANAME) AS TABLE_SCHEM,  \
 		LOWER (CAST (TABLENAME AS VARCHAR(128))) AS TABLE_NAME, \
 		LOWER (CASE WHEN TABLETYPE = 'VIEW' THEN 'VIEW' \
-		     ELSE CAST ((TABLETYPE || ' TABLE') AS VARCHAR(16)) END) AS TABLE_TYPE, \
+		     ELSE CAST (('TABLE') AS VARCHAR(16)) END) AS TABLE_TYPE, \
 		CAST ('' AS VARCHAR(128)) AS REMARKS, \
 		CAST (NULL AS VARCHAR(128)) AS TYPE_CAT, \
 		CAST (NULL AS VARCHAR(128)) AS TYPE_SCHEM, \

diff --git a/...d/shared/src/main/java/com/pivotal/gemfirexd/internal/shared/common/reference/Limits.java b/...d/shared/src/main/java/com/pivotal/gemfirexd/internal/shared/common/reference/Limits.java
@@ -58,7 +58,7 @@ public interface Limits {
    * Trigger, Column, Schema, Savepoint, Table and View names)
    * are limited to 128 */
   public static final int MAX_IDENTIFIER_LENGTH = 128;
-  public static final int DB2_CHAR_MAXWIDTH = 254;
+  public static final int DB2_CHAR_MAXWIDTH = 255;
   public static final int DB2_VARCHAR_MAXWIDTH = 32672;
   public static final int DB2_LOB_MAXWIDTH = 2147483647;
   public static final int DB2_LONGVARCHAR_MAXWIDTH = 32700;