From e3e1743a85e8e3be9e1d9f970e938b73aabd1c57 Mon Sep 17 00:00:00 2001
From: Sunjeet Singh <sunjeets@netflix.com>
Date: Sat, 7 Sep 2024 07:01:49 -0700
Subject: [PATCH] Tests for set and map type state resharding

---
 ...llowListTypeDataElementsSplitJoinTest.java |  4 +-
 .../list/HollowListTypeReadStateTest.java     |  4 +-
 ...llowListTypeDataElementsSplitJoinTest.java |  2 +-
 ...ollowMapTypeDataElementsSplitJoinTest.java | 32 ++++++++++---
 ...ollowMapTypeDataElementsSplitJoinTest.java | 12 ++---
 .../map/HollowMapTypeReadStateTest.java       | 47 +++++++++++++++++++
 ...ollowMapTypeDataElementsSplitJoinTest.java |  2 +-
 ...owObjectTypeDataElementsSplitJoinTest.java |  2 +-
 ...ollowSetTypeDataElementsSplitJoinTest.java | 11 +++++
 .../set/HollowSetTypeReadStateTest.java       | 47 +++++++++++++++++++
 10 files changed, 141 insertions(+), 22 deletions(-)
 create mode 100644 hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java
 create mode 100644 hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java

diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java
index 610e95915..424c801dd 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java
@@ -19,7 +19,7 @@ public void testSplitThenJoin() throws IOException {
         int maxNumListRecords = 100;
 
         // 1->2->1, 1->4->1, ...
-        for (int numRecords=0;numRecords<maxNumListRecords;numRecords++) { // SNAP: TODO: what is this iteration for?
+        for (int numRecords=0;numRecords<maxNumListRecords;numRecords++) {
 
             int[][] listContents = generateListContents(numRecords);
             HollowListTypeReadState typeReadState = populateTypeStateWith(listContents);
@@ -27,7 +27,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(numRecords, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState, listContents);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowListTypeDataElementsSplitter splitter = new HollowListTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowListTypeDataElements[] splitElements = splitter.split();
 
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java
index 158458c65..71694bb41 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java
@@ -11,9 +11,9 @@ public class HollowListTypeReadStateTest extends AbstractHollowListTypeDataEleme
     @Test
     public void testResharding() throws Exception {
 
-        for (int shardingFactor : new int[]{2, 4, 8, 16, 32, 64, 128, 256, 512, 1024})
+        for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024
         {
-            for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100))
+            for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000))
             {
                 int[][] listContents = generateListContents(numRecords);
                 HollowListTypeReadState listTypeReadState = populateTypeStateWith(listContents);
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/VMSHollowListTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/VMSHollowListTypeDataElementsSplitJoinTest.java
index e364d69b7..992653dee 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/VMSHollowListTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/VMSHollowListTypeDataElementsSplitJoinTest.java
@@ -34,7 +34,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(numListRecords, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState, listContents);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowListTypeDataElementsSplitter splitter = new HollowListTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowListTypeDataElements[] splitElements = splitter.split();
 
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java
index 76e6f2844..e1658ecf4 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java
@@ -12,6 +12,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 import org.junit.Before;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
@@ -42,13 +43,15 @@ protected void initializeTypeStates() {
     private void populateWriteStateEngine(int[][][] maps) {
         // populate state so that there is 1:1 correspondence in key/value ordinal to value in int type
         // find max value across all maps
-        int numKeyValueOrdinals = 1 + Arrays.stream(maps)
-                .flatMap(Arrays::stream)
-                .flatMapToInt(Arrays::stream)
-                .max()
-                .orElseThrow(() -> new IllegalArgumentException("Array is empty"));
-        // populate write state with that many ordinals
-        super.populateWriteStateEngine(numKeyValueOrdinals);
+        if (maps.length > 0) {
+            int numKeyValueOrdinals = 1 + Arrays.stream(maps)
+                    .flatMap(Arrays::stream)
+                    .flatMapToInt(Arrays::stream)
+                    .max()
+                    .orElseThrow(() -> new IllegalArgumentException("Array is empty"));
+            // populate write state with that many ordinals
+            super.populateWriteStateEngine(numKeyValueOrdinals);
+        }
         for(int[][] map : maps) {
             HollowMapWriteRecord rec = new HollowMapWriteRecord();
             for (int[] entry : map) {
@@ -65,6 +68,21 @@ protected HollowMapTypeReadState populateTypeStateWith(int[][][] maps) throws IO
         return (HollowMapTypeReadState) readStateEngine.getTypeState("TestMap");
     }
 
+    protected int[][][] generateListContents(int numRecords) {
+        int[][][] maps = new int[numRecords][][];
+        Random random = new Random();
+        int maxEntries = 10;
+        for (int i=0;i<numRecords;i++) {
+            int numEntries = 1 + random.nextInt(maxEntries);
+            maps[i] = new int[numEntries][2];
+            for (int j=0;j<numEntries;j++) {
+                maps[i][j][0] = (i * maxEntries) + j;
+                maps[i][j][1] = (i * maxEntries) + j + 1;
+            }
+        }
+        return maps;
+    }
+
     protected void assertDataUnchanged(HollowMapTypeReadState typeState, int[][][] maps) {
         int numMapRecords = maps.length;
         for(int i=0;i<numMapRecords;i++) {
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitJoinTest.java
index 4bfccfa6e..a4a1b5ec6 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitJoinTest.java
@@ -15,21 +15,17 @@ public class HollowMapTypeDataElementsSplitJoinTest extends AbstractHollowMapTyp
 
     @Test
     public void testSplitThenJoin() throws IOException {
-        int[][][] maps = new int[][][] {
-                { {33321, 1}, {2, 2}, {32224, 3} },
-                { {1, 31442}, {2, 1}, {3, 2} },
-                { {1002, 2} },
-                { {0, 134} },
-        };
+        int maxNumMapRecords = 100;
 
         // 1->2->1, 1->4->1, ...
-        for (int listRecord=0;listRecord<maps.length;listRecord++) {    // SNAP: TODO: what is this iteration for?
+        for (int numRecords=0;numRecords<maxNumMapRecords;numRecords++) {
+            int[][][] maps = generateListContents(numRecords);
             HollowMapTypeReadState typeReadState = populateTypeStateWith(maps);
             assertEquals(1, typeReadState.numShards());
             assertEquals(maps.length, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState,maps);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowMapTypeDataElementsSplitter splitter = new HollowMapTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowMapTypeDataElements[] splitElements = splitter.split();
 
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java
new file mode 100644
index 000000000..4ba341e8a
--- /dev/null
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java
@@ -0,0 +1,47 @@
+package com.netflix.hollow.core.read.engine.map;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.netflix.hollow.core.read.engine.HollowTypeReshardingStrategy;
+import java.util.Random;
+import org.junit.Test;
+
+public class HollowMapTypeReadStateTest extends AbstractHollowMapTypeDataElementsSplitJoinTest {
+
+    @Test
+    public void testResharding() throws Exception {
+
+        for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024
+        {
+            for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000))
+            {
+                int[][][] listContents = generateListContents(numRecords);
+                HollowMapTypeReadState mapTypeReadState = populateTypeStateWith(listContents);
+                assertDataUnchanged(mapTypeReadState, listContents);
+                HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(mapTypeReadState);
+
+                // Splitting shards
+                {
+                    int prevShardCount = mapTypeReadState.numShards();
+                    int newShardCount = shardingFactor * prevShardCount;
+                    reshardingStrategy.reshard(mapTypeReadState, mapTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, mapTypeReadState.numShards());
+                    assertEquals(newShardCount, shardingFactor * prevShardCount);
+                }
+                assertDataUnchanged(mapTypeReadState, listContents);
+
+                // Joining shards
+                {
+                    int prevShardCount = mapTypeReadState.numShards();
+                    int newShardCount = prevShardCount / shardingFactor;
+                    reshardingStrategy.reshard(mapTypeReadState, mapTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, mapTypeReadState.numShards());
+                    assertEquals(shardingFactor * newShardCount, prevShardCount);
+                }
+                assertDataUnchanged(mapTypeReadState, listContents);
+            }
+        }
+    }
+}
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/VMSHollowMapTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/VMSHollowMapTypeDataElementsSplitJoinTest.java
index 0ff0141fa..d7b95e2ad 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/VMSHollowMapTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/VMSHollowMapTypeDataElementsSplitJoinTest.java
@@ -29,7 +29,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(maps.length, typeReadState.getPopulatedOrdinals().cardinality());
             assertDataUnchanged(typeReadState,maps);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowMapTypeDataElementsSplitter splitter = new HollowMapTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowMapTypeDataElements[] splitElements = splitter.split();
 
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java
index af1e675ab..6f795c48c 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java
@@ -21,7 +21,7 @@ public void testSplitThenJoin() throws IOException {
             assertEquals(1, typeReadState.numShards());
             assertDataUnchanged(typeReadState, numRecords);
 
-            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}) {
+            for (int numSplits : new int[]{1, 2, 4, 8, 16, 32}) {  // , 64, 128, 256, 512, 1024
                 HollowObjectTypeDataElementsSplitter splitter = new HollowObjectTypeDataElementsSplitter(typeReadState.currentDataElements()[0], numSplits);
                 HollowObjectTypeDataElements[] splitElements = splitter.split();
                 HollowObjectTypeDataElementsJoiner joiner = new HollowObjectTypeDataElementsJoiner(splitElements);
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java
index 7c679116c..78f84d719 100644
--- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/AbstractHollowSetTypeDataElementsSplitJoinTest.java
@@ -41,6 +41,17 @@ protected void initializeTypeStates() {
         writeStateEngine.setTargetMaxTypeShardSize(4 * 100 * 1000 * 1024);
     }
 
+    int[][] generateListContents(int numRecords) {
+        int[][] listContents = new int[numRecords][];
+        for (int i=0;i<numRecords;i++) {
+            listContents[i] = new int[i+1];
+            for (int j=0;j<i+1;j++) {
+                listContents[i][j] = j;
+            }
+        }
+        return listContents;
+    }
+
     protected HollowSetTypeReadState populateTypeStateWith(int[][] setContents) throws IOException {
         int numOrdinals = 1 + Arrays.stream(setContents)
                 .flatMapToInt(Arrays::stream)
diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java
new file mode 100644
index 000000000..33c1b27ac
--- /dev/null
+++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java
@@ -0,0 +1,47 @@
+package com.netflix.hollow.core.read.engine.set;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.netflix.hollow.core.read.engine.HollowTypeReshardingStrategy;
+import java.util.Random;
+import org.junit.Test;
+
+public class HollowSetTypeReadStateTest extends AbstractHollowSetTypeDataElementsSplitJoinTest {
+
+    @Test
+    public void testResharding() throws Exception {
+
+        for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024
+        {
+            for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100))
+            {
+                int[][] listContents = generateListContents(numRecords);
+                HollowSetTypeReadState setTypeReadState = populateTypeStateWith(listContents);
+                assertDataUnchanged(setTypeReadState, listContents);
+                HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(setTypeReadState);
+
+                // Splitting shards
+                {
+                    int prevShardCount = setTypeReadState.numShards();
+                    int newShardCount = shardingFactor * prevShardCount;
+                    reshardingStrategy.reshard(setTypeReadState, setTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, setTypeReadState.numShards());
+                    assertEquals(newShardCount, shardingFactor * prevShardCount);
+                }
+                assertDataUnchanged(setTypeReadState, listContents);
+
+                // Joining shards
+                {
+                    int prevShardCount = setTypeReadState.numShards();
+                    int newShardCount = prevShardCount / shardingFactor;
+                    reshardingStrategy.reshard(setTypeReadState, setTypeReadState.numShards(), newShardCount);
+
+                    assertEquals(newShardCount, setTypeReadState.numShards());
+                    assertEquals(shardingFactor * newShardCount, prevShardCount);
+                }
+                assertDataUnchanged(setTypeReadState, listContents);
+            }
+        }
+    }
+}