Tests for set and map type state resharding

Netflix · Sep 7, 2024 · e3e1743 · e3e1743
1 parent 9e377a4
commit e3e1743
Show file tree

Hide file tree

Showing 10 changed files with 141 additions and 22 deletions.
diff --git a/...ava/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java b/...ava/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java
@@ -19,15 +19,15 @@ public void testSplitThenJoin() throws IOException {
         int maxNumListRecords = 100;
 
         // 1->2->1, 1->4->1, ...
-        for (int numRecords=0;numRecords<maxNumListRecords;numRecords++) { // SNAP: TODO: what is this iteration for?
+        for (int numRecords=0;numRecords<maxNumListRecords;numRecords++) {
 
             int[][] listContents = generateListContents(numRecords);
             HollowListTypeReadState typeReadState = populateTypeStateWith(listContents);
             assertEquals(1, typeReadState.numShards());
             assertEquals(numRecords, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState, listContents);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowListTypeDataElementsSplitter splitter = new HollowListTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowListTypeDataElements[] splitElements = splitter.split();
 

diff --git a/...w/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java b/...w/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java
@@ -11,9 +11,9 @@ public class HollowListTypeReadStateTest extends AbstractHollowListTypeDataEleme
     @Test
     public void testResharding() throws Exception {
 
-        for (int shardingFactor : new int[]{2, 4, 8, 16, 32, 64, 128, 256, 512, 1024})
+        for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024
         {
-            for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100))
+            for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000))
             {
                 int[][] listContents = generateListContents(numRecords);
                 HollowListTypeReadState listTypeReadState = populateTypeStateWith(listContents);

diff --git a/.../com/netflix/hollow/core/read/engine/list/VMSHollowListTypeDataElementsSplitJoinTest.java b/.../com/netflix/hollow/core/read/engine/list/VMSHollowListTypeDataElementsSplitJoinTest.java
@@ -34,7 +34,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(numListRecords, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState, listContents);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowListTypeDataElementsSplitter splitter = new HollowListTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowListTypeDataElements[] splitElements = splitter.split();
 

diff --git a/...m/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java b/...m/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java
@@ -12,6 +12,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 import org.junit.Before;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
@@ -42,13 +43,15 @@ protected void initializeTypeStates() {
     private void populateWriteStateEngine(int[][][] maps) {
         // populate state so that there is 1:1 correspondence in key/value ordinal to value in int type
         // find max value across all maps
-        int numKeyValueOrdinals = 1 + Arrays.stream(maps)
-                .flatMap(Arrays::stream)
-                .flatMapToInt(Arrays::stream)
-                .max()
-                .orElseThrow(() -> new IllegalArgumentException("Array is empty"));
-        // populate write state with that many ordinals
-        super.populateWriteStateEngine(numKeyValueOrdinals);
+        if (maps.length > 0) {
+            int numKeyValueOrdinals = 1 + Arrays.stream(maps)
+                    .flatMap(Arrays::stream)
+                    .flatMapToInt(Arrays::stream)
+                    .max()
+                    .orElseThrow(() -> new IllegalArgumentException("Array is empty"));
+            // populate write state with that many ordinals
+            super.populateWriteStateEngine(numKeyValueOrdinals);
+        }
         for(int[][] map : maps) {
             HollowMapWriteRecord rec = new HollowMapWriteRecord();
             for (int[] entry : map) {
@@ -65,6 +68,21 @@ protected HollowMapTypeReadState populateTypeStateWith(int[][][] maps) throws IO
         return (HollowMapTypeReadState) readStateEngine.getTypeState("TestMap");
     }
 
+    protected int[][][] generateListContents(int numRecords) {
+        int[][][] maps = new int[numRecords][][];
+        Random random = new Random();
+        int maxEntries = 10;
+        for (int i=0;i<numRecords;i++) {
+            int numEntries = 1 + random.nextInt(maxEntries);
+            maps[i] = new int[numEntries][2];
+            for (int j=0;j<numEntries;j++) {
+                maps[i][j][0] = (i * maxEntries) + j;
+                maps[i][j][1] = (i * maxEntries) + j + 1;
+            }
+        }
+        return maps;
+    }
+
     protected void assertDataUnchanged(HollowMapTypeReadState typeState, int[][][] maps) {
         int numMapRecords = maps.length;
         for(int i=0;i<numMapRecords;i++) {

diff --git a/.../java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitJoinTest.java b/.../java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitJoinTest.java
@@ -15,21 +15,17 @@ public class HollowMapTypeDataElementsSplitJoinTest extends AbstractHollowMapTyp
 
     @Test
     public void testSplitThenJoin() throws IOException {
-        int[][][] maps = new int[][][] {
-                { {33321, 1}, {2, 2}, {32224, 3} },
-                { {1, 31442}, {2, 1}, {3, 2} },
-                { {1002, 2} },
-                { {0, 134} },
-        };
+        int maxNumMapRecords = 100;
 
         // 1->2->1, 1->4->1, ...
-        for (int listRecord=0;listRecord<maps.length;listRecord++) {    // SNAP: TODO: what is this iteration for?
+        for (int numRecords=0;numRecords<maxNumMapRecords;numRecords++) {
+            int[][][] maps = generateListContents(numRecords);
             HollowMapTypeReadState typeReadState = populateTypeStateWith(maps);
             assertEquals(1, typeReadState.numShards());
             assertEquals(maps.length, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState,maps);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowMapTypeDataElementsSplitter splitter = new HollowMapTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowMapTypeDataElements[] splitElements = splitter.split();
 

diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java
@@ -0,0 +1,47 @@
+package com.netflix.hollow.core.read.engine.map;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.netflix.hollow.core.read.engine.HollowTypeReshardingStrategy;
+import java.util.Random;
+import org.junit.Test;
+
+public class HollowMapTypeReadStateTest extends AbstractHollowMapTypeDataElementsSplitJoinTest {
+
+    @Test
+    public void testResharding() throws Exception {
+
+        for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024
+        {
+            for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000))
+            {
+                int[][][] listContents = generateListContents(numRecords);
+                HollowMapTypeReadState mapTypeReadState = populateTypeStateWith(listContents);
+                assertDataUnchanged(mapTypeReadState, listContents);
+                HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(mapTypeReadState);
+
+                // Splitting shards
+                {
+                    int prevShardCount = mapTypeReadState.numShards();
+                    int newShardCount = shardingFactor * prevShardCount;
+                    reshardingStrategy.reshard(mapTypeReadState, mapTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, mapTypeReadState.numShards());
+                    assertEquals(newShardCount, shardingFactor * prevShardCount);
+                }
+                assertDataUnchanged(mapTypeReadState, listContents);
+
+                // Joining shards
+                {
+                    int prevShardCount = mapTypeReadState.numShards();
+                    int newShardCount = prevShardCount / shardingFactor;
+                    reshardingStrategy.reshard(mapTypeReadState, mapTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, mapTypeReadState.numShards());
+                    assertEquals(shardingFactor * newShardCount, prevShardCount);
+                }
+                assertDataUnchanged(mapTypeReadState, listContents);
+            }
+        }
+    }
+}
diff --git a/...va/com/netflix/hollow/core/read/engine/map/VMSHollowMapTypeDataElementsSplitJoinTest.java b/...va/com/netflix/hollow/core/read/engine/map/VMSHollowMapTypeDataElementsSplitJoinTest.java
@@ -29,7 +29,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(maps.length, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState,maps);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowMapTypeDataElementsSplitter splitter = new HollowMapTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowMapTypeDataElements[] splitElements = splitter.split();
 

diff --git a/...com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java b/...com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java
@@ -21,7 +21,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(1, typeReadState.numShards());
             assertDataUnchanged(typeReadState, numRecords);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowObjectTypeDataElementsSplitter splitter = new HollowObjectTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowObjectTypeDataElements[] splitElements = splitter.split();
                 HollowObjectTypeDataElementsJoiner joiner = new HollowObjectTypeDataElementsJoiner(splitElements);

diff --git a/...m/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java b/...m/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java
@@ -41,6 +41,17 @@ protected void initializeTypeStates() {
         writeStateEngine.setTargetMaxTypeShardSize(4 * 100 * 1000 * 1024);
     }
 
+    int[][] generateListContents(int numRecords) {
+        int[][] listContents = new int[numRecords][];
+        for (int i=0;i<numRecords;i++) {
+            listContents[i] = new int[i+1];
+            for (int j=0;j<i+1;j++) {
+                listContents[i][j] = j;
+            }
+        }
+        return listContents;
+    }
+
     protected HollowSetTypeReadState populateTypeStateWith(int[][] setContents) throws IOException {
         int numOrdinals = 1 + Arrays.stream(setContents)
                 .flatMapToInt(Arrays::stream)

diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java
@@ -0,0 +1,47 @@
+package com.netflix.hollow.core.read.engine.set;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.netflix.hollow.core.read.engine.HollowTypeReshardingStrategy;
+import java.util.Random;
+import org.junit.Test;
+
+public class HollowSetTypeReadStateTest extends AbstractHollowSetTypeDataElementsSplitJoinTest {
+
+    @Test
+    public void testResharding() throws Exception {
+
+        for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024
+        {
+            for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100))
+            {
+                int[][] listContents = generateListContents(numRecords);
+                HollowSetTypeReadState setTypeReadState = populateTypeStateWith(listContents);
+                assertDataUnchanged(setTypeReadState, listContents);
+                HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(setTypeReadState);
+
+                // Splitting shards
+                {
+                    int prevShardCount = setTypeReadState.numShards();
+                    int newShardCount = shardingFactor * prevShardCount;
+                    reshardingStrategy.reshard(setTypeReadState, setTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, setTypeReadState.numShards());
+                    assertEquals(newShardCount, shardingFactor * prevShardCount);
+                }
+                assertDataUnchanged(setTypeReadState, listContents);
+
+                // Joining shards
+                {
+                    int prevShardCount = setTypeReadState.numShards();
+                    int newShardCount = prevShardCount / shardingFactor;
+                    reshardingStrategy.reshard(setTypeReadState, setTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, setTypeReadState.numShards());
+                    assertEquals(shardingFactor * newShardCount, prevShardCount);
+                }
+                assertDataUnchanged(setTypeReadState, listContents);
+            }
+        }
+    }
+}