diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java new file mode 100644 index 000000000..91eae74d4 --- /dev/null +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/VMSHollowTypeReshardingStrategyTest.java @@ -0,0 +1,43 @@ +package com.netflix.hollow.core.read.engine; + +import static junit.framework.TestCase.assertEquals; + +import com.netflix.hollow.api.consumer.HollowConsumer; +import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever; +import com.netflix.hollow.tools.checksum.HollowChecksum; +import java.nio.file.Paths; +import org.junit.Test; + +public class VMSHollowTypeReshardingStrategyTest { + + @Test + public void testSplittingAndJoiningWithSnapshotBlob() throws Exception { + + String blobPath = "/Users/ssingh/workspace/blob-cache/vms-daintree/prod/"; // null; // dir where snapshot blob exists for e.g. "/tmp/"; + long v = 20230611133921525l; // 0l; // snapshot version for e.g. 20230915162636001l; + int[] shardingFactorArray = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; // + + HollowFilesystemBlobRetriever hollowBlobRetriever = new HollowFilesystemBlobRetriever(Paths.get(blobPath)); + HollowConsumer c = HollowConsumer + .withBlobRetriever(hollowBlobRetriever).build(); + c.triggerRefreshTo(v); + + for (HollowTypeReadState typeReadState : c.getStateEngine().getTypeStates()) { + for (int shardingFactor : shardingFactorArray) { + if (blobPath==null || v==0l) { + throw new IllegalArgumentException("These arguments need to be specified"); + } + + HollowChecksum origChecksum = typeReadState.getChecksum(typeReadState.getSchema()); + + HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(typeReadState); + reshardingStrategy.reshard(typeReadState, typeReadState.numShards(), shardingFactor * typeReadState.numShards()); + reshardingStrategy.reshard(typeReadState, typeReadState.numShards(), typeReadState.numShards() / shardingFactor); + HollowChecksum joinedChecksum = typeReadState.getChecksum(typeReadState.getSchema()); + + assertEquals(joinedChecksum, origChecksum); + System.out.println("Processed type " + typeReadState.getSchema().getName() + " with " + typeReadState.numShards() + " shard and sharding factor " + shardingFactor); + } + } + } +} diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java index 71694bb41..2e63af4f4 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java @@ -11,9 +11,9 @@ public class HollowListTypeReadStateTest extends AbstractHollowListTypeDataEleme @Test public void testResharding() throws Exception { - for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024 + for (int shardingFactor : new int[]{2, 4, 8}) // , 16, 32, 64, 128, 256, 512, 1024 { - for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000)) + for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100)) { int[][] listContents = generateListContents(numRecords); HollowListTypeReadState listTypeReadState = populateTypeStateWith(listContents); diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java index 4ba341e8a..7bc68c19e 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadStateTest.java @@ -11,9 +11,9 @@ public class HollowMapTypeReadStateTest extends AbstractHollowMapTypeDataElement @Test public void testResharding() throws Exception { - for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024 + for (int shardingFactor : new int[]{2, 4, 8}) // , 16, 32, 64, 128, 256, 512, 1024 { - for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000)) + for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100)) { int[][][] listContents = generateListContents(numRecords); HollowMapTypeReadState mapTypeReadState = populateTypeStateWith(listContents); diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeReadStateTest.java index b08b23399..5b2a93d86 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeReadStateTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeReadStateTest.java @@ -10,7 +10,7 @@ public class HollowObjectTypeReadStateTest extends AbstractHollowObjectTypeDataE @Test public void testResharding() throws Exception { - for (int shardingFactor : new int[]{2, 4, 8, 16}) // 32, 64, 128, 256, 512, 1024... + for (int shardingFactor : new int[]{2, 4, 8}) // , 16, 32, 64, 128, 256, 512, 1024... { for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000)) { diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java index 33c1b27ac..2084abc83 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadStateTest.java @@ -11,7 +11,7 @@ public class HollowSetTypeReadStateTest extends AbstractHollowSetTypeDataElement @Test public void testResharding() throws Exception { - for (int shardingFactor : new int[]{2, 4, 8, 16}) // , 32, 64, 128, 256, 512, 1024 + for (int shardingFactor : new int[]{2, 4, 8}) // , 16, 32, 64, 128, 256, 512, 1024 { for(int numRecords=1;numRecords<=1000;numRecords+=new Random().nextInt(100)) {