Support hive-2.3.9

.circleci/config.yml

@@ -64,11 +64,11 @@ custom_filters:
   merge_only: &merge_only
     filters:
       branches:
-        only: hive-1.2.1000.2.6.4.0-91
+        only: hive-2.3.9
   merge_ignore: &merge_ignore
     filters:
       branches:
-        ignore: hive-1.2.1000.2.6.4.0-91
+        ignore: hive-2.3.9
 
 workflows:
   build-and-test:

pom.xml

@@ -15,7 +15,7 @@
 
   <groupId>jp.co.yahoo.yosegi</groupId>
   <artifactId>yosegi-hive</artifactId>
-  <version>2.0.7_hive-1.2.1000.2.6.4.0-91-SNAPSHOT</version>
+  <version>2.0.0_hive-2.3.9-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Yosegi Hive</name>
   <description>Yosegi Hive library.</description>
@@ -54,8 +54,6 @@
   </developers>
 
   <properties>
-    <configlibVersion>1.2.1.1</configlibVersion>
-    <schemalibVersion>1.2.4_hive-1.2.1000.2.6.4.0-91</schemalibVersion>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <target_jdk_version>1.8</target_jdk_version>
     <maven-surefire-plugin.version>3.0.0-M2</maven-surefire-plugin.version>
@@ -64,6 +62,8 @@
     <buildtype>test</buildtype>
     <spotbugs-maven-plugin.version>3.1.8</spotbugs-maven-plugin.version>
     <spotbugs.version>3.1.9</spotbugs.version>
+    <hive.base>2.3</hive.base>
+    <hive.version>${hive.base}.9</hive.version>
   </properties>
 
   <distributionManagement>
@@ -87,7 +87,7 @@
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-exec</artifactId>
-      <version>1.2.1000.2.6.4.0-91</version>
+      <version>${hive.version}</version>
       <optional>true</optional>
       <exclusions>
         <exclusion>
@@ -106,7 +106,7 @@
     <dependency>
       <groupId>jp.co.yahoo.yosegi</groupId>
       <artifactId>yosegi</artifactId>
-      <version>2.0.2</version>
+      <version>2.0.9</version>
     </dependency>
     <dependency>
       <groupId>jp.co.yahoo.yosegi</groupId>
@@ -217,6 +217,7 @@
         <artifactId>apache-rat-plugin</artifactId>
         <configuration>
           <excludes>
+            <exclude>jars/**/*</exclude>
             <exclude>**/*.json</exclude>
             <exclude>**/*.txt</exclude>
             <exclude>**/*.template</exclude>
@@ -253,8 +254,8 @@
         <version>2.5.3</version>
         <configuration>
           <releaseProfiles>gpg-sign</releaseProfiles>
-          <releaseVersion>${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}_hive-1.2.1000.2.6.4.0-91</releaseVersion>
-          <developmentVersion>${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.nextIncrementalVersion}_hive-1.2.1000.2.6.4.0-91-SNAPSHOT</developmentVersion>
+          <releaseVersion>${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}_hive-${hive.version}</releaseVersion>
+          <developmentVersion>${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.nextIncrementalVersion}_hive-${hive.version}-SNAPSHOT</developmentVersion>
         </configuration>
       </plugin>
       <plugin>

src/main/java/jp/co/yahoo/yosegi/hive/YosegiObjectInspectorFactory.java

@@ -58,6 +58,10 @@ public static ObjectInspector craeteObjectInspectorFromTypeInfo( final TypeInfo
         switch ( primitiveTypeInfo.getPrimitiveCategory() ) {
           case STRING:
             return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+          case CHAR:
+            return PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector;
+          case VARCHAR:
+            return PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector;
           case BINARY:
             return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
           case BOOLEAN:
@@ -74,10 +78,12 @@ public static ObjectInspector craeteObjectInspectorFromTypeInfo( final TypeInfo
             return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
           case SHORT:
             return PrimitiveObjectInspectorFactory.writableShortObjectInspector;
-
+          case TIMESTAMP:
+            return PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
           case DATE:
+            return PrimitiveObjectInspectorFactory.writableDateObjectInspector;
+
           case DECIMAL:
-          case TIMESTAMP:
           case VOID:
           default:
             throw new UnsupportedOperationException(

src/main/java/jp/co/yahoo/yosegi/hive/io/HiveReaderSetting.java

@@ -23,8 +23,7 @@
 import jp.co.yahoo.yosegi.spread.expression.IExpressionNode;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -34,7 +33,6 @@
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.JobConf;
 
-import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -76,11 +74,10 @@ public HiveReaderSetting( final FileSplit split, final JobConf job ) {
     disableSkipBlock = job.getBoolean( "yosegi.disable.block.skip" , false );
     disableFilterPushdown = job.getBoolean( "yosegi.disable.filter.pushdown" , false );
 
-    Set<String> pathNameSet = createPathSet( split.getPath() );
     List<ExprNodeGenericFuncDesc> filterExprs = new ArrayList<ExprNodeGenericFuncDesc>();
     String filterExprSerialized = job.get( TableScanDesc.FILTER_EXPR_CONF_STR );
     if ( filterExprSerialized != null ) {
-      filterExprs.add( Utilities.deserializeExpression(filterExprSerialized) );
+      filterExprs.add( SerializationUtilities.deserializeExpression( filterExprSerialized ) );
     }
     config.set( "spread.reader.read.column.names" , createReadColumnNames(
         job.get( ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR , null ) ) );
@@ -117,11 +114,8 @@ public HiveReaderSetting( final FileSplit split, final JobConf job ) {
       isVectorModeFlag = false;
       return;
     } else {
-      for ( Map.Entry<String,PartitionDesc> pathsAndParts
+      for ( Map.Entry<Path,PartitionDesc> pathsAndParts
           : mapWork.getPathToPartitionInfo().entrySet() ) {
-        if ( ! pathNameSet.contains( pathsAndParts.getKey() ) ) {
-          continue;
-        }
         Properties props = pathsAndParts.getValue().getTableDesc().getProperties();
         if ( props.containsKey( "yosegi.expand" ) ) {
           config.set( "spread.reader.expand.column" , props.getProperty( "yosegi.expand" ) );
@@ -140,9 +134,7 @@ public HiveReaderSetting( final FileSplit split, final JobConf job ) {
 
     node = createExpressionNode( filterExprs );
 
-    // Next Hive vesion;
-    // Utilities.getUseVectorizedInputFileFormat(job)
-    isVectorModeFlag = Utilities.isVectorMode( job );
+    isVectorModeFlag = Utilities.getUseVectorizedInputFileFormat( job );
   }
 
   /**

src/main/java/jp/co/yahoo/yosegi/hive/io/PrimitiveToWritableConverter.java

@@ -22,8 +22,12 @@
 import jp.co.yahoo.yosegi.message.objects.PrimitiveObject;
 import jp.co.yahoo.yosegi.spread.column.ICell;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.BytesWritable;
@@ -34,6 +38,8 @@
 import org.apache.hadoop.io.Writable;
 
 import java.io.IOException;
+import java.sql.Date;
+import java.sql.Timestamp;
 
 public final class PrimitiveToWritableConverter {
 
@@ -62,6 +68,14 @@ public static Writable convert(
           textResult.set( strBytes , 0 , strBytes.length );
         }
         return textResult;
+      case CHAR:
+        HiveCharWritable charResult = new HiveCharWritable();
+        charResult.set( primitiveObject.getString() );
+        return charResult;
+      case VARCHAR:
+        HiveVarcharWritable varcharResult = new HiveVarcharWritable();
+        varcharResult.set( primitiveObject.getString() );
+        return varcharResult;
       case BINARY:
         BytesWritable bytesResult = new BytesWritable();
         byte[] bytes = primitiveObject.getBytes();
@@ -119,9 +133,23 @@ public static Writable convert(
           return null;
         }
         return doubleResult;
+      case TIMESTAMP:
+        TimestampWritable timestampResult = new TimestampWritable();
+        try {
+          timestampResult.set( new Timestamp( primitiveObject.getLong() ) );
+        } catch ( NumberFormatException | NullPointerException ex ) {
+          return null;
+        }
+        return timestampResult;
       case DATE:
+        DateWritable dateResult = new DateWritable();
+        try {
+          dateResult.set( new Date( primitiveObject.getLong() ) );
+        } catch ( NumberFormatException | NullPointerException ex ) {
+          return null;
+        }
+        return dateResult;
       case DECIMAL:
-      case TIMESTAMP:
       case VOID:
       default:
         throw new UnsupportedOperationException( "Unknown category " + primitiveCategory );

src/main/java/jp/co/yahoo/yosegi/hive/io/YosegiHiveParserOutputFormat.java

@@ -77,6 +77,10 @@ public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecor
       String compressionClass = tableProperties.getProperty( "yosegi.compression.class" );
       config.set( "spread.column.maker.default.compress.class" , compressionClass );
     }
+    if ( tableProperties.containsKey( "yosegi.compress.optimize.allowed.ratio" ) ) {
+      String allowedRatio = tableProperties.getProperty( "yosegi.compress.optimize.allowed.ratio" );
+      config.set( "spread.optimize.allowed.ratio" , allowedRatio );
+    }
     FileSystem fs = outputPath.getFileSystem( job );
     long dfsBlockSize = Math.max( fs.getDefaultBlockSize( outputPath ) , 1024 * 1024 * 256 );
     OutputStream out = fs.create(

src/main/java/jp/co/yahoo/yosegi/message/parser/hive/HiveCharPrimitiveConverter.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.yahoo.yosegi.message.parser.hive;
+
+import jp.co.yahoo.yosegi.message.objects.NullObj;
+import jp.co.yahoo.yosegi.message.objects.PrimitiveObject;
+import jp.co.yahoo.yosegi.message.objects.StringObj;
+
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+
+import java.io.IOException;
+
+public class HiveCharPrimitiveConverter implements IHivePrimitiveConverter {
+
+  private final HiveCharObjectInspector inspector;
+
+  public HiveCharPrimitiveConverter( final HiveCharObjectInspector inspector ) {
+    this.inspector = inspector;
+  }
+
+  @Override
+  public PrimitiveObject get(final Object target ) throws IOException {
+    if ( target == null ) {
+      return NullObj.getInstance();
+    }
+    return new StringObj( inspector.getPrimitiveWritableObject( target ).toString() );
+  }
+
+}

src/main/java/jp/co/yahoo/yosegi/message/parser/hive/HiveDatePrimitiveConverter.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.yahoo.yosegi.message.parser.hive;
+
+import jp.co.yahoo.yosegi.message.objects.LongObj;
+import jp.co.yahoo.yosegi.message.objects.NullObj;
+import jp.co.yahoo.yosegi.message.objects.PrimitiveObject;
+
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+
+import java.io.IOException;
+
+public class HiveDatePrimitiveConverter implements IHivePrimitiveConverter {
+
+  private final DateObjectInspector inspector;
+
+  public HiveDatePrimitiveConverter( final DateObjectInspector inspector ) {
+    this.inspector = inspector;
+  }
+
+  @Override
+  public PrimitiveObject get(final Object target ) throws IOException {
+    if ( target == null ) {
+      return NullObj.getInstance();
+    }
+    return new LongObj( inspector.getPrimitiveJavaObject( target ).getTime() );
+  }
+
+}

src/main/java/jp/co/yahoo/yosegi/message/parser/hive/HivePrimitiveConverterFactory.java

@@ -23,12 +23,16 @@
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
 
 public class HivePrimitiveConverterFactory {
 
@@ -47,6 +51,8 @@ public static IHivePrimitiveConverter get( final ObjectInspector objectInspector
             return new HiveBooleanPrimitiveConverter( (BooleanObjectInspector)objectInspector );
           case BYTE:
             return new HiveBytePrimitiveConverter( (ByteObjectInspector)objectInspector );
+          case CHAR:
+            return new HiveCharPrimitiveConverter( (HiveCharObjectInspector)objectInspector );
           case DOUBLE:
             return new HiveDoublePrimitiveConverter( (DoubleObjectInspector)objectInspector );
           case FLOAT:
@@ -59,8 +65,12 @@ public static IHivePrimitiveConverter get( final ObjectInspector objectInspector
             return new HiveShortPrimitiveConverter( (ShortObjectInspector)objectInspector );
           case STRING:
             return new HiveStringPrimitiveConverter( (StringObjectInspector)objectInspector );
+          case VARCHAR:
+            return new HiveVarcharPrimitiveConverter( (HiveVarcharObjectInspector)objectInspector );
           case DATE:
+            return new HiveDatePrimitiveConverter( (DateObjectInspector)objectInspector );
           case TIMESTAMP:
+            return new HiveTimestampPrimitiveConverter( (TimestampObjectInspector)objectInspector );
           case VOID:
           case UNKNOWN:
           default: