diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java index c274f2a00..5364323a5 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java @@ -239,7 +239,7 @@ static void copyRecord(HollowObjectTypeDataElements to, int toOrdinal, HollowObj : from.fixedLengthData.getElementValue(currentReadFixedLengthStartBit, from.bitsPerField[fieldIndex]); long toWriteFixedLengthStartBit = ((long)toOrdinal * to.bitsPerRecord) + to.bitOffsetPerField[fieldIndex]; - if(to.varLengthData[fieldIndex] == null) { + if(to.varLengthData[fieldIndex] == null) { // fixed len data if(readValue == from.nullValueForField[fieldIndex]) { writeNullFixedLengthField(to, fieldIndex, toWriteFixedLengthStartBit); } @@ -247,10 +247,8 @@ static void copyRecord(HollowObjectTypeDataElements to, int toOrdinal, HollowObj to.fixedLengthData.setElementValue(toWriteFixedLengthStartBit, to.bitsPerField[fieldIndex], readValue); } } else { - if ((readValue & (1L << (from.bitsPerField[fieldIndex] - 1))) != 0) { - // SNAP: TODO: also nulls for test for float, double (special bit sequences), bytes, + if ((readValue & (1L << (from.bitsPerField[fieldIndex] - 1))) != 0) { // null check is the first bit set (other bits have an offset of the last non-null value) writeNullVarLengthField(to, fieldIndex, toWriteFixedLengthStartBit, currentWriteVarLengthDataPointers); - // SNAP: TODO: Maybe refactor: writeNullField(to, fieldIndex, toWriteFixedLengthStartBit, currentWriteVarLengthDataPointers); } else { long fromStartByte = varLengthStartByte(from, fromOrdinal, fieldIndex); long fromEndByte = varLengthEndByte(from, fromOrdinal, fieldIndex); diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitJoinTest.java similarity index 95% rename from hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeDataElementsSplitJoinTest.java rename to hollow/src/test/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitJoinTest.java index fd055efc4..b75d776a8 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitJoinTest.java @@ -6,7 +6,7 @@ import com.netflix.hollow.core.write.HollowObjectWriteRecord; import org.junit.Before; -public class HollowTypeDataElementsSplitJoinTest extends AbstractStateEngineTest { +public class AbstractHollowTypeDataElementsSplitJoinTest extends AbstractStateEngineTest { protected HollowObjectSchema schema; @Before diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java index 66e7443eb..34f452a56 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java @@ -15,23 +15,16 @@ public void testSplittingAndJoiningWithSnapshotBlob() throws Exception { String blobPath = "/Users/ssingh/workspace/blob-cache/vms-daintree/prod/"; // null; // dir where snapshot blob exists for e.g. "/tmp/"; long v = 20230611133921525l; // 0l; // snapshot version for e.g. 20230915162636001l; - int[] shardingFactorArray = {2 , 4, 8, 16, 32, 64, 128, 256, 512, 1024}; // + int[] shardingFactorArray = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; // HollowFilesystemBlobRetriever hollowBlobRetriever = new HollowFilesystemBlobRetriever(Paths.get(blobPath)); HollowConsumer c = HollowConsumer .withBlobRetriever(hollowBlobRetriever).build(); c.triggerRefreshTo(v); - // SNAP: TODO: object type splitter is making null value an empty string (for inline string, not sure if it matters), // could be that the null value needs to be recomputed for the split shard? for (HollowTypeReadState typeReadState : c.getStateEngine().getTypeStates()) { for (int shardingFactor : shardingFactorArray) { - if (!(typeReadState.getSchema().getName().equals("StreamVariant") - || typeReadState.getSchema().getName().equals("Certification") - || typeReadState.getSchema().getName().equals("Phase") - || typeReadState.getSchema().getName().equals("EncodingProfile"))) { - continue; - } System.out.println("Processing type " + typeReadState.getSchema().getName() + " with " + typeReadState.numShards() + " shard and sharding factor " + shardingFactor); if (blobPath==null || v==0l) { throw new IllegalArgumentException("These arguments need to be specified"); diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java index 097246c3e..58d298292 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java @@ -2,7 +2,7 @@ import static org.mockito.Mockito.when; -import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest; +import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest; import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator; import com.netflix.hollow.core.schema.HollowListSchema; import com.netflix.hollow.core.write.HollowListTypeWriteState; @@ -14,7 +14,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -public class AbstractHollowListTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest { +public class AbstractHollowListTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest { protected HollowListSchema listSchema; @Mock diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java index e1658ecf4..a7230f251 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java @@ -3,7 +3,7 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; -import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest; +import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest; import com.netflix.hollow.core.read.iterator.HollowMapEntryOrdinalIterator; import com.netflix.hollow.core.schema.HollowMapSchema; import com.netflix.hollow.core.write.HollowMapTypeWriteState; @@ -17,7 +17,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -public class AbstractHollowMapTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest { +public class AbstractHollowMapTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest { protected HollowMapSchema mapSchema; @Mock diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java index 8f4a0566d..8042976ba 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java @@ -4,7 +4,7 @@ import static org.mockito.Mockito.when; import com.netflix.hollow.api.objects.generic.GenericHollowObject; -import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest; +import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.read.filter.HollowFilterConfig; import com.netflix.hollow.core.util.StateEngineRoundTripper; @@ -13,7 +13,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -public class AbstractHollowObjectTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest { +public class AbstractHollowObjectTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest { @Mock protected HollowObjectTypeReadState mockObjectTypeState; diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java index 6f795c48c..84ae6c2cf 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java @@ -6,6 +6,7 @@ import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.schema.HollowSchema; +import com.netflix.hollow.core.write.HollowObjectWriteRecord; import com.netflix.hollow.tools.checksum.HollowChecksum; import java.io.IOException; import java.nio.file.Paths; @@ -85,6 +86,39 @@ public void testSplitThenJoinWithEmptyJoin() throws IOException { assertEquals(-1, joined.maxOrdinal); } + @Test + public void testSplitThenJoinWithNullAndSpecialValues() throws IOException { + initWriteStateEngine(); + HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema); + for(int i=0;i<10;i++) { + rec.reset(); + rec.setLong("longField", i); + // other fields will be null + writeStateEngine.add("TestObject", rec); + } + for(int i=10;i<20;i++) { + rec.reset(); + rec.setLong("longField", Long.MIN_VALUE); + rec.setString("stringField", ""); + rec.setInt("intField", i); + rec.setDouble("doubleField", Double.NaN); + writeStateEngine.add("TestObject", rec); + } + + roundTripSnapshot(); + HollowObjectTypeReadState typeReadState = (HollowObjectTypeReadState) readStateEngine.getTypeState("TestObject"); + assertEquals(1, typeReadState.numShards()); + + HollowObjectTypeDataElementsSplitter splitter = new HollowObjectTypeDataElementsSplitter(typeReadState.currentDataElements()[0], 4); + HollowObjectTypeDataElements[] splitBy4 = splitter.split(); + + HollowObjectTypeDataElementsJoiner joiner = new HollowObjectTypeDataElementsJoiner(splitBy4); + HollowObjectTypeDataElements joined = joiner.join(); + + HollowObjectTypeReadState joinedTypeReadState = new HollowObjectTypeReadState(typeReadState.getSchema(), joined); + assertChecksumUnchanged(typeReadState, joinedTypeReadState, typeReadState.getPopulatedOrdinals()); + } + // manually invoked // @Test public void testSplittingAndJoiningWithSnapshotBlob() throws Exception { diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java index 78f84d719..c108f5196 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java @@ -2,7 +2,7 @@ import static org.mockito.Mockito.when; -import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest; +import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest; import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator; import com.netflix.hollow.core.schema.HollowSetSchema; import com.netflix.hollow.core.write.HollowSetTypeWriteState; @@ -17,7 +17,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -public class AbstractHollowSetTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest { +public class AbstractHollowSetTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest { protected HollowSetSchema setSchema; @Mock