Add test for object types null values handling

Netflix · Sep 8, 2024 · 6a68c76 · 6a68c76
1 parent e1a4c62
commit 6a68c76
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 21 deletions.
diff --git a/...rc/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java b/...rc/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java
@@ -239,18 +239,16 @@ static void copyRecord(HollowObjectTypeDataElements to, int toOrdinal, HollowObj
                     : from.fixedLengthData.getElementValue(currentReadFixedLengthStartBit, from.bitsPerField[fieldIndex]);
 
             long toWriteFixedLengthStartBit = ((long)toOrdinal * to.bitsPerRecord) + to.bitOffsetPerField[fieldIndex];
-            if(to.varLengthData[fieldIndex] == null) {
+            if(to.varLengthData[fieldIndex] == null) {   // fixed len data
                 if(readValue == from.nullValueForField[fieldIndex]) {
                     writeNullFixedLengthField(to, fieldIndex, toWriteFixedLengthStartBit);
                 }
                 else {
                     to.fixedLengthData.setElementValue(toWriteFixedLengthStartBit, to.bitsPerField[fieldIndex], readValue);
                 }
             } else {
-                if ((readValue & (1L << (from.bitsPerField[fieldIndex] - 1))) != 0) {
-                    // SNAP: TODO: also nulls for test for float, double (special bit sequences), bytes,
+                if ((readValue & (1L << (from.bitsPerField[fieldIndex] - 1))) != 0) {   // null check is the first bit set (other bits have an offset of the last non-null value)
                     writeNullVarLengthField(to, fieldIndex, toWriteFixedLengthStartBit, currentWriteVarLengthDataPointers);
-                    // SNAP: TODO: Maybe refactor: writeNullField(to, fieldIndex, toWriteFixedLengthStartBit, currentWriteVarLengthDataPointers);
                 } else {
                     long fromStartByte = varLengthStartByte(from, fromOrdinal, fieldIndex);
                     long fromEndByte = varLengthEndByte(from, fromOrdinal, fieldIndex);

diff --git a/.../HollowTypeDataElementsSplitJoinTest.java → ...tHollowTypeDataElementsSplitJoinTest.java b/.../HollowTypeDataElementsSplitJoinTest.java → ...tHollowTypeDataElementsSplitJoinTest.java
@@ -6,7 +6,7 @@
 import com.netflix.hollow.core.write.HollowObjectWriteRecord;
 import org.junit.Before;
 
-public class HollowTypeDataElementsSplitJoinTest extends AbstractStateEngineTest {
+public class AbstractHollowTypeDataElementsSplitJoinTest extends AbstractStateEngineTest {
     protected HollowObjectSchema schema;
 
     @Before

diff --git a/...rc/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java b/...rc/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java
@@ -15,23 +15,16 @@ public void testSplittingAndJoiningWithSnapshotBlob() throws Exception {
 
         String blobPath = "/Users/ssingh/workspace/blob-cache/vms-daintree/prod/"; // null; // dir where snapshot blob exists for e.g. "/tmp/";
         long v = 20230611133921525l; // 0l; // snapshot version for e.g. 20230915162636001l;
-        int[] shardingFactorArray = {2 , 4, 8, 16, 32, 64, 128, 256, 512, 1024}; //
+        int[] shardingFactorArray = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; //
 
         HollowFilesystemBlobRetriever hollowBlobRetriever = new HollowFilesystemBlobRetriever(Paths.get(blobPath));
         HollowConsumer c = HollowConsumer
                 .withBlobRetriever(hollowBlobRetriever).build();
         c.triggerRefreshTo(v);
 
-        // SNAP: TODO: object type splitter is making null value an empty string (for inline string, not sure if it matters),
         // could be that the null value needs to be recomputed for the split shard?
         for (HollowTypeReadState typeReadState : c.getStateEngine().getTypeStates()) {
             for (int shardingFactor : shardingFactorArray) {
-                if (!(typeReadState.getSchema().getName().equals("StreamVariant")
-                 || typeReadState.getSchema().getName().equals("Certification")
-                || typeReadState.getSchema().getName().equals("Phase")
-                 || typeReadState.getSchema().getName().equals("EncodingProfile"))) {
-                    continue;
-                }
                 System.out.println("Processing type " + typeReadState.getSchema().getName() + " with " + typeReadState.numShards() + " shard and sharding factor " + shardingFactor);
                 if (blobPath==null || v==0l) {
                     throw new IllegalArgumentException("These arguments need to be specified");

diff --git a/...netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java b/...netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java
@@ -2,7 +2,7 @@
 
 import static org.mockito.Mockito.when;
 
-import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest;
+import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
 import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
 import com.netflix.hollow.core.schema.HollowListSchema;
 import com.netflix.hollow.core.write.HollowListTypeWriteState;
@@ -14,7 +14,7 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
-public class AbstractHollowListTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest {
+public class AbstractHollowListTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest {
     protected HollowListSchema listSchema;
 
     @Mock

diff --git a/...m/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java b/...m/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java
@@ -3,7 +3,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.when;
 
-import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest;
+import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
 import com.netflix.hollow.core.read.iterator.HollowMapEntryOrdinalIterator;
 import com.netflix.hollow.core.schema.HollowMapSchema;
 import com.netflix.hollow.core.write.HollowMapTypeWriteState;
@@ -17,7 +17,7 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
-public class AbstractHollowMapTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest {
+public class AbstractHollowMapTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest {
     protected HollowMapSchema mapSchema;
 
     @Mock

diff --git a/...lix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java b/...lix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java
@@ -4,7 +4,7 @@
 import static org.mockito.Mockito.when;
 
 import com.netflix.hollow.api.objects.generic.GenericHollowObject;
-import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest;
+import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
 import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
 import com.netflix.hollow.core.read.filter.HollowFilterConfig;
 import com.netflix.hollow.core.util.StateEngineRoundTripper;
@@ -13,7 +13,7 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
-public class AbstractHollowObjectTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest {
+public class AbstractHollowObjectTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest {
 
     @Mock
     protected HollowObjectTypeReadState mockObjectTypeState;

diff --git a/...com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java b/...com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java
@@ -6,6 +6,7 @@
 import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever;
 import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
 import com.netflix.hollow.core.schema.HollowSchema;
+import com.netflix.hollow.core.write.HollowObjectWriteRecord;
 import com.netflix.hollow.tools.checksum.HollowChecksum;
 import java.io.IOException;
 import java.nio.file.Paths;
@@ -85,6 +86,39 @@ public void testSplitThenJoinWithEmptyJoin() throws IOException {
         assertEquals(-1, joined.maxOrdinal);
     }
 
+    @Test
+    public void testSplitThenJoinWithNullAndSpecialValues() throws IOException {
+        initWriteStateEngine();
+        HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
+        for(int i=0;i<10;i++) {
+            rec.reset();
+            rec.setLong("longField", i);
+            // other fields will be null
+            writeStateEngine.add("TestObject", rec);
+        }
+        for(int i=10;i<20;i++) {
+            rec.reset();
+            rec.setLong("longField", Long.MIN_VALUE);
+            rec.setString("stringField", "");
+            rec.setInt("intField", i);
+            rec.setDouble("doubleField", Double.NaN);
+            writeStateEngine.add("TestObject", rec);
+        }
+
+        roundTripSnapshot();
+        HollowObjectTypeReadState typeReadState = (HollowObjectTypeReadState) readStateEngine.getTypeState("TestObject");
+        assertEquals(1, typeReadState.numShards());
+
+        HollowObjectTypeDataElementsSplitter splitter = new HollowObjectTypeDataElementsSplitter(typeReadState.currentDataElements()[0], 4);
+        HollowObjectTypeDataElements[] splitBy4 = splitter.split();
+
+        HollowObjectTypeDataElementsJoiner joiner = new HollowObjectTypeDataElementsJoiner(splitBy4);
+        HollowObjectTypeDataElements joined = joiner.join();
+
+        HollowObjectTypeReadState joinedTypeReadState = new HollowObjectTypeReadState(typeReadState.getSchema(), joined);
+        assertChecksumUnchanged(typeReadState, joinedTypeReadState, typeReadState.getPopulatedOrdinals());
+    }
+
     // manually invoked
     // @Test
     public void testSplittingAndJoiningWithSnapshotBlob() throws Exception {

diff --git a/...m/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java b/...m/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java
@@ -2,7 +2,7 @@
 
 import static org.mockito.Mockito.when;
 
-import com.netflix.hollow.core.read.engine.HollowTypeDataElementsSplitJoinTest;
+import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
 import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
 import com.netflix.hollow.core.schema.HollowSetSchema;
 import com.netflix.hollow.core.write.HollowSetTypeWriteState;
@@ -17,7 +17,7 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
-public class AbstractHollowSetTypeDataElementsSplitJoinTest extends HollowTypeDataElementsSplitJoinTest {
+public class AbstractHollowSetTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest {
     protected HollowSetSchema setSchema;
 
     @Mock