Merge pull request yahoojapan#108 from koijima/Yosegi-91

Yosegi-91 Shorten the setting string of Flatten.

pom.xml

@@ -16,7 +16,7 @@
 
   <groupId>jp.co.yahoo.yosegi</groupId>
   <artifactId>yosegi</artifactId>
-  <version>1.0.6</version>
+  <version>1.1.0</version>
   <packaging>jar</packaging>
   <name>Yosegi</name>
   <description>Yosegi package.</description>

src/main/java/jp/co/yahoo/yosegi/binary/maker/OptimizedNullArrayCellManager.java

@@ -97,8 +97,8 @@ public PrimitiveObject[] getPrimitiveObjectArray(
       final int length ) {
     PrimitiveObject[] result = new PrimitiveObject[length];
     int loopEnd = ( start + length );
-    if ( size() < loopEnd ) {
-      loopEnd = size();
+    if ( indexList.size() < loopEnd ) {
+      loopEnd = indexList.size();
     }
     for ( int i = start,index = 0 ; i < loopEnd ; i++,index++ ) {
       int valueIndex = indexList.get( i );
@@ -119,16 +119,19 @@ public void setPrimitiveObjectArray(
       final int length ,
       final IMemoryAllocator allocator ) {
     int loopEnd = ( start + length );
-    if ( size() < loopEnd ) {
-      loopEnd = size();
+    if ( indexList.size() < loopEnd ) {
+      loopEnd = indexList.size();
     }
     int index = 0;
     for ( int i = start ; i < loopEnd ; i++,index++ ) {
-      if ( i < startIndex || valueArray[ i - startIndex ] == null ) {
+      int valueIndex = indexList.get( i );
+      if ( valueIndex < startIndex
+          || size() <= valueIndex
+          || valueArray[ valueIndex - startIndex ] == null ) {
         allocator.setNull( index );
       } else {
         try {
-          allocator.setPrimitiveObject( index , valueArray[i - startIndex] );
+          allocator.setPrimitiveObject( index , valueArray[valueIndex - startIndex] );
         } catch ( IOException ex ) {
           throw new UncheckedIOException( ex );
         }

src/main/java/jp/co/yahoo/yosegi/binary/maker/OptimizedNullArrayDicCellManager.java

@@ -103,14 +103,17 @@ public PrimitiveObject[] getPrimitiveObjectArray(
       final int length ) {
     PrimitiveObject[] result = new PrimitiveObject[length];
     int loopEnd = ( start + length );
-    if ( size() < loopEnd ) {
-      loopEnd = size();
+    if ( indexList.size() < loopEnd ) {
+      loopEnd = indexList.size();
     }
     for ( int i = start , index = 0 ; i < loopEnd ; i++,index++ ) {
-      if ( i < startIndex || isNullArray[ i - startIndex ] ) {
+      int valueIndex = indexList.get( i );
+      if ( valueIndex < startIndex
+          || size() <= valueIndex 
+          || isNullArray[ valueIndex - startIndex ] ) {
         continue;
       }
-      result[index] = dicArray[dicIndexArray[i - startIndex]];
+      result[index] = dicArray[ dicIndexArray[ valueIndex - startIndex ] ];
     }
     return result;
   }
@@ -122,16 +125,20 @@ public void setPrimitiveObjectArray(
       final int length ,
       final IMemoryAllocator allocator ) {
     int loopEnd = ( start + length );
-    if ( size() < loopEnd ) {
-      loopEnd = size();
+    if ( indexList.size() < loopEnd ) {
+      loopEnd = indexList.size();
     }
     int index = 0;
     for ( int i = start ; i < loopEnd ; i++,index++ ) {
-      if ( i < startIndex || isNullArray[ i - startIndex ] ) {
+      int valueIndex = indexList.get( i );
+      if ( valueIndex < startIndex
+          || size() <= valueIndex
+          || isNullArray[ valueIndex - startIndex ] ) {
         allocator.setNull( index );
       } else {
         try {
-          allocator.setPrimitiveObject( index , dicArray[dicIndexArray[i - startIndex]] );
+          allocator.setPrimitiveObject(
+              index , dicArray[dicIndexArray[ valueIndex - startIndex] ] );
         } catch ( IOException ex ) {
           throw new UncheckedIOException( ex );
         }

src/main/java/jp/co/yahoo/yosegi/spread/flatten/FlattenFunction.java

@@ -31,7 +31,13 @@ public class FlattenFunction implements IFlattenFunction {
 
   private final List<FlattenColumn> flattenColumnList = new ArrayList<FlattenColumn>();
   private final Map<String,FlattenColumn> flattenColumnMap = new HashMap<String,FlattenColumn>();
+  private final Map<String,Integer> duplicateKeyMap = new HashMap<String,Integer>();
   private final List<String> filterColumnList = new ArrayList<String>();
+  private String delimiter = "_";
+
+  public void setDelimiter( final String delimiter ) {
+    this.delimiter = delimiter;
+  }
 
   public boolean isEmpty() {
     return flattenColumnList.isEmpty();
@@ -48,6 +54,25 @@ public void add( final FlattenColumn flattenColumn ) {
     if ( ! flattenColumnMap.containsKey( flattenColumn.getLinkName() ) ) {
       flattenColumnList.add( flattenColumn );
       flattenColumnMap.put( flattenColumn.getLinkName() , flattenColumn );
+    } else {
+      if ( ! duplicateKeyMap.containsKey( flattenColumn.getLinkName() ) ) {
+        duplicateKeyMap.put( flattenColumn.getLinkName() , Integer.valueOf( 0 ) );
+      }
+      int num = duplicateKeyMap.get( flattenColumn.getLinkName() ).intValue();
+      String newLinkKey = String.format(
+          "%s%s%d" ,
+          flattenColumn.getLinkName() ,
+          delimiter ,
+          num );
+      duplicateKeyMap.put( flattenColumn.getLinkName() , Integer.valueOf( num + 1 ) );
+      if ( flattenColumnMap.containsKey( newLinkKey ) ) {
+        add( flattenColumn );
+      } else {
+        FlattenColumn newFlattenColumn = new FlattenColumn(
+            newLinkKey , flattenColumn.getFilterColumnNameArray() );
+        flattenColumnList.add( newFlattenColumn );
+        flattenColumnMap.put( newLinkKey , newFlattenColumn );
+      }
     }
   }
 

src/main/java/jp/co/yahoo/yosegi/spread/flatten/FlattenFunctionFactory.java

@@ -21,8 +21,15 @@
 import jp.co.yahoo.yosegi.config.Configuration;
 import jp.co.yahoo.yosegi.message.parser.IParser;
 import jp.co.yahoo.yosegi.message.parser.json.JacksonMessageReader;
+import jp.co.yahoo.yosegi.util.replacement.IReplacement;
+import jp.co.yahoo.yosegi.util.replacement.PrefixAndSuffix;
+import jp.co.yahoo.yosegi.util.replacement.ReplacementFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
 
 public final class FlattenFunctionFactory {
 
@@ -32,23 +39,58 @@ private FlattenFunctionFactory() {}
    * Create flattening process from Configuration.
    */
   public static IFlattenFunction get( final Configuration config ) throws IOException {
-    String flattenJson = config.get( "spread.reader.flatten.column" , null );
-    if ( flattenJson == null || flattenJson.isEmpty() ) {
-      return new NotFlattenFunction();
+    Iterator<String> iterator = config.getKey().iterator();
+    List<String> targetKeyNameList = new ArrayList<String>();
+    while ( iterator.hasNext() ) {
+      String keyName = iterator.next();
+      if ( ! keyName.startsWith( "spread.reader.flatten.column" ) ) {
+        continue;
+      }
+      String flattenJson = config.get( keyName , null );
+      if ( flattenJson == null ) {
+        continue;
+      }
+      targetKeyNameList.add( keyName );
     }
-    JacksonMessageReader jsonReader = new JacksonMessageReader();
-    IParser jsonParser = jsonReader.create( flattenJson );
 
-    FlattenFunction flattenFunction = parseJson( jsonParser );
-    if ( flattenFunction.isEmpty() ) {
+    if ( targetKeyNameList.isEmpty() ) {
       return new NotFlattenFunction();
     }
+    Collections.sort( targetKeyNameList );
+
+    FlattenFunction flattenFunction = new FlattenFunction();
+    for ( String keyName : targetKeyNameList ) {
+      String flattenJson = config.get( keyName , null );
+      JacksonMessageReader jsonReader = new JacksonMessageReader();
+      IParser jsonParser = jsonReader.create( flattenJson );
+      parseJson( jsonParser , flattenFunction );
+    }
 
     return flattenFunction;
   }
 
-  private static FlattenFunction parseJson( final IParser jsonParser ) throws IOException {
-    FlattenFunction flattenFunction = new FlattenFunction();
+  private static void parseJson(
+      final IParser jsonParser ,
+      final FlattenFunction flattenFunction ) throws IOException {
+    if ( ! jsonParser.containsKey( "version" )
+        || jsonParser.get( "version" ).getString().isEmpty() ) {
+      parseJsonV1( jsonParser , flattenFunction );
+      return;
+    }
+    String version = jsonParser.get( "version" ).getString();
+    switch ( version ) {
+      case "2":
+        parseJsonV2( jsonParser , flattenFunction );
+        return;
+      default:
+        throw new IOException(
+            String.format( "The Flatten configuration version %s is incorrect." , version ) );
+    }
+  }
+
+  private static void parseJsonV1(
+      final IParser jsonParser ,
+      final FlattenFunction flattenFunction ) throws IOException {
     for ( int i = 0 ; i < jsonParser.size() ; i++ ) {
       IParser flattenColumnParser = jsonParser.getParser(i);
       String linkName = flattenColumnParser.get( "link_name" ).getString();
@@ -62,8 +104,39 @@ private static FlattenFunction parseJson( final IParser jsonParser ) throws IOEx
       }
       flattenFunction.add( new FlattenColumn( linkName , nodeArray ) );
     }
+  }
 
-    return flattenFunction;
+  private static void parseJsonV2(
+      final IParser jsonParser ,
+      final FlattenFunction flattenFunction ) throws IOException {
+    String version = jsonParser.get( "replacement" ).getString();
+    IReplacement replacement = ReplacementFactory.get( version );
+    IParser flattenSettingParser = jsonParser.getParser( "flatten" );
+    for ( int i = 0 ; i < flattenSettingParser.size() ; i++ ) {
+      IParser flattenColumnParser = flattenSettingParser.getParser(i);
+      IParser nodeParser = flattenColumnParser.getParser( "column" );
+      String[] nodeArray = new String[ nodeParser.size() ];
+      for ( int n = 0 ; n < nodeParser.size() ; n++ ) {
+        nodeArray[n] = nodeParser.get(n).getString();
+      }
+      String prefix = flattenColumnParser.get( "prefix" ).getString();
+      String suffix = flattenColumnParser.get( "suffix" ).getString();
+      String delimiter = flattenColumnParser.get( "delimiter" ).getString();
+      if ( delimiter == null ) {
+        delimiter = "_";
+      }
+      flattenFunction.setDelimiter( delimiter );
+      PrefixAndSuffix prefixAndSuffixAppender = new PrefixAndSuffix( prefix , suffix , delimiter );
+      IParser targetParser = flattenColumnParser.getParser( "target" );
+      for ( int n = 0 ; n < targetParser.size() ; n++ ) {
+        String[] childNodeArray = new String[ nodeArray.length + 1 ];
+        childNodeArray[ childNodeArray.length - 1 ] = targetParser.get(n).getString();
+        System.arraycopy( nodeArray , 0 , childNodeArray , 0 , nodeArray.length );
+        String linkName = prefixAndSuffixAppender.append(
+            replacement.replace( childNodeArray[ childNodeArray.length - 1 ] ) );
+        flattenFunction.add( new FlattenColumn( linkName , childNodeArray ) );
+      }
+    }
   }
 
 }

src/main/java/jp/co/yahoo/yosegi/util/replacement/DefaultReplacement.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.yahoo.yosegi.util.replacement;
+
+public class DefaultReplacement implements IReplacement {
+
+  @Override
+  public String replace( final String str ) {
+    return str;
+  }
+
+}

src/main/java/jp/co/yahoo/yosegi/util/replacement/IPrefix.java

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.yahoo.yosegi.util.replacement;
+
+public interface IPrefix {
+
+  String appendPrefix( final String str );
+
+}

src/main/java/jp/co/yahoo/yosegi/util/replacement/IReplacement.java

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.yahoo.yosegi.util.replacement;
+
+public interface IReplacement {
+
+  String replace( final String str );
+
+}

src/main/java/jp/co/yahoo/yosegi/util/replacement/LowerReplacement.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.yahoo.yosegi.util.replacement;
+
+public class LowerReplacement implements IReplacement {
+
+  @Override
+  public String replace( final String str ) {
+    return str.toLowerCase();
+  }
+
+}