From 9e377a40dcdb96a6ce85817086a5ab254c80d2d3 Mon Sep 17 00:00:00 2001 From: Sunjeet Singh Date: Sat, 7 Sep 2024 06:23:39 -0700 Subject: [PATCH] add test for List type resharding --- .../core/read/engine/HollowBlobReader.java | 2 +- .../engine/HollowTypeReshardingStrategy.java | 7 +- .../list/HollowListTypeShardsHolder.java | 3 +- .../engine/map/HollowMapTypeShardsHolder.java | 3 +- .../engine/set/HollowSetTypeShardsHolder.java | 3 +- .../HollowTypeReshardingStrategyTest.java | 196 ++++++++++++++++++ ...llowListTypeDataElementsSplitJoinTest.java | 11 + ...llowListTypeDataElementsSplitJoinTest.java | 15 +- .../list/HollowListTypeReadStateTest.java | 47 +++++ ...ollowMapTypeDataElementsSplitJoinTest.java | 2 +- .../object/HollowObjectTypeReadStateTest.java | 154 +------------- .../core/write/HollowTypeWriteStateTest.java | 1 - 12 files changed, 275 insertions(+), 169 deletions(-) create mode 100644 hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategyTest.java create mode 100644 hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateTest.java diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowBlobReader.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowBlobReader.java index 6d970cf470..050a99febf 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowBlobReader.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowBlobReader.java @@ -373,7 +373,7 @@ private String readTypeStateDelta(HollowBlobInput in) throws IOException { if(typeState != null) { if (shouldReshard(typeState, typeState.numShards(), numShards)) { HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(typeState); - reshardingStrategy.reshard(typeState, numShards); + reshardingStrategy.reshard(typeState, typeState.numShards(), numShards); } typeState.applyDelta(in, schema, stateEngine.getMemoryRecycler(), numShards); } else { diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategy.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategy.java index fafe2d155e..3ed09434be 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategy.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategy.java @@ -38,10 +38,11 @@ public static HollowTypeReshardingStrategy getInstance(HollowTypeReadState typeS * Reshards this type state to the desired shard count using O(shard size) space while supporting concurrent reads * into the underlying data elements. * - * @param newNumShards The desired number of shards + * @param typeState The type state to reshard + * @param prevNumShards The current number of shards in typeState + * @param newNumShards The desired number of shards for typeState */ - public void reshard(HollowTypeReadState typeState, int newNumShards) { - int prevNumShards = typeState.getShardsVolatile().getShards().length; // SNAP: TODO: or .shards.length + public void reshard(HollowTypeReadState typeState, int prevNumShards, int newNumShards) { int shardingFactor = shardingFactor(prevNumShards, newNumShards); HollowTypeDataElements[] newDataElements; int[] shardOrdinalShifts; diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeShardsHolder.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeShardsHolder.java index e81108e5ff..a3f0ac015e 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeShardsHolder.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeShardsHolder.java @@ -42,8 +42,7 @@ public HollowListTypeShardsHolder(HollowTypeReadStateShard[] fromShards) { @Override public HollowTypeReadStateShard[] getShards() { - throw new UnsupportedOperationException("Not implemented yet"); - // return shards; + return shards; } @Override diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeShardsHolder.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeShardsHolder.java index 1ef42922b1..4205d3034f 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeShardsHolder.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeShardsHolder.java @@ -42,8 +42,7 @@ public HollowMapTypeShardsHolder(HollowTypeReadStateShard[] fromShards) { @Override public HollowTypeReadStateShard[] getShards() { - throw new UnsupportedOperationException("Not implemented yet"); - // return shards; + return shards; } @Override diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeShardsHolder.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeShardsHolder.java index f52c5faf35..e9a999cff7 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeShardsHolder.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeShardsHolder.java @@ -42,8 +42,7 @@ public HollowSetTypeShardsHolder(HollowTypeReadStateShard[] fromShards) { @Override public HollowTypeReadStateShard[] getShards() { - throw new UnsupportedOperationException("Not implemented yet"); - // return shards; + return shards; } @Override diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategyTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategyTest.java new file mode 100644 index 0000000000..6559badd57 --- /dev/null +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/HollowTypeReshardingStrategyTest.java @@ -0,0 +1,196 @@ +package com.netflix.hollow.core.read.engine; + +import static com.netflix.hollow.core.read.engine.HollowTypeReshardingStrategy.shardingFactor; +import static junit.framework.TestCase.assertEquals; + +import com.netflix.hollow.api.objects.generic.GenericHollowObject; +import com.netflix.hollow.core.memory.MemoryMode; +import com.netflix.hollow.core.read.engine.object.HollowObjectTypeDataElements; +import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState; +import com.netflix.hollow.core.read.engine.object.HollowObjectTypeShardsHolder; +import com.netflix.hollow.core.schema.HollowObjectSchema; +import com.netflix.hollow.core.util.StateEngineRoundTripper; +import com.netflix.hollow.core.write.HollowObjectTypeWriteState; +import com.netflix.hollow.core.write.HollowObjectWriteRecord; +import com.netflix.hollow.core.write.HollowWriteStateEngine; +import java.io.IOException; +import java.util.Random; +import java.util.function.Supplier; +import org.junit.Assert; +import org.junit.Test; + +public class HollowTypeReshardingStrategyTest { + + private HollowReadStateEngine readStateEngine; + private HollowObjectSchema schema; + + @Test + public void testMappingAnOrdinalToAShardAndBack() { + int maxOrdinal = 1000; + int numShards = 4; + int minRecordLocationsPerShard = (maxOrdinal + 1) / numShards; + int[][] shardOrdinals = new int[numShards][]; + for(int i=0;i> shardOrdinalShift; + shardOrdinals[shardIndex][shardOrdinal] = ordinal; + } + + for (int shardIndex=0; shardIndex shardingFactor(0, 1)); + assertIllegalStateException(() -> shardingFactor(2, 0)); + assertIllegalStateException(() -> shardingFactor(1, 1)); + assertIllegalStateException(() -> shardingFactor(1, -1)); + assertIllegalStateException(() -> shardingFactor(2, 3)); + } + + @Test + public void testReshardingIntermediateStages_expandWithOriginalDataElements() throws Exception { + for (int shardingFactor : new int[]{2, 4}) { + for(int numRecords=1;numRecords<=100000;numRecords+=new Random().nextInt(5000)) + { + HollowObjectTypeReadState expectedTypeState = populateTypeStateWith(numRecords); + HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(expectedTypeState); + + HollowObjectTypeShardsHolder original = expectedTypeState.getShardsVolatile(); + HollowObjectTypeReadState actualTypeState = new HollowObjectTypeReadState(readStateEngine, MemoryMode.ON_HEAP, schema, schema); + actualTypeState.updateShardsVolatile(reshardingStrategy.expandWithOriginalDataElements(original, shardingFactor)); + + assertEquals(shardingFactor * expectedTypeState.numShards(), actualTypeState.numShards()); + assertDataUnchanged(actualTypeState, numRecords); + } + } + } + + @Test + public void testReshardingIntermediateStages_splitDataElementsForOneShard() throws Exception { + for (int shardingFactor : new int[]{2, 4}) { + for(int numRecords=1;numRecords<=100000;numRecords+=new Random().nextInt(5000)) + { + HollowObjectTypeReadState typeState = populateTypeStateWith(numRecords); + HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(typeState); + + HollowObjectTypeShardsHolder originalShardsHolder = typeState.getShardsVolatile(); + int originalNumShards = typeState.numShards(); + + // expand shards + typeState.updateShardsVolatile(reshardingStrategy.expandWithOriginalDataElements(originalShardsHolder, shardingFactor)); + + for(int i=0; i invocation) { + try { + invocation.get(); + Assert.fail(); + } catch (IllegalStateException e) { + // expected + } + } + +} diff --git a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java index fd8774d666..097246c3e8 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java @@ -63,6 +63,17 @@ protected HollowListTypeReadState populateTypeStateWith(int[][] listContents) th return (HollowListTypeReadState) readStateEngine.getTypeState("TestList"); } + protected int[][] generateListContents(int numRecords) { + int[][] listContents = new int[numRecords][]; + for (int i=0;i2->1, 1->4->1, ... - for (int listRecord=0;listRecord2->1, 1->4->1, ... - for (int listRecord=0;listRecord> shardOrdinalShift; - shardOrdinals[shardIndex][shardOrdinal] = ordinal; - } - - for (int shardIndex=0; shardIndex shardingFactor(0, 1)); - assertIllegalStateException(() -> shardingFactor(2, 0)); - assertIllegalStateException(() -> shardingFactor(1, 1)); - assertIllegalStateException(() -> shardingFactor(1, -1)); - assertIllegalStateException(() -> shardingFactor(2, 3)); - } - @Test public void testResharding() throws Exception { for (int shardingFactor : new int[]{2, 4, 8, 16}) // 32, 64, 128, 256, 512, 1024... { - for(int numRecords=1;numRecords<=100000;numRecords+=new Random().nextInt(1000)) + for(int numRecords=1;numRecords<=10000;numRecords+=new Random().nextInt(1000)) { HollowObjectTypeReadState objectTypeReadState = populateTypeStateWith(numRecords); assertDataUnchanged(objectTypeReadState, numRecords); + HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(objectTypeReadState); // Splitting shards { int prevShardCount = objectTypeReadState.numShards(); int newShardCount = shardingFactor * prevShardCount; - RESHARDING_STRATEGY.reshard(objectTypeReadState, newShardCount); + reshardingStrategy.reshard(objectTypeReadState, objectTypeReadState.numShards(), newShardCount); assertEquals(newShardCount, objectTypeReadState.numShards()); assertEquals(newShardCount, shardingFactor * prevShardCount); @@ -91,7 +33,7 @@ public void testResharding() throws Exception { { int prevShardCount = objectTypeReadState.numShards(); int newShardCount = prevShardCount / shardingFactor; - RESHARDING_STRATEGY.reshard(objectTypeReadState, newShardCount); + reshardingStrategy.reshard(objectTypeReadState, objectTypeReadState.numShards(), newShardCount); assertEquals(newShardCount, objectTypeReadState.numShards()); assertEquals(shardingFactor * newShardCount, prevShardCount); @@ -109,12 +51,13 @@ public void testReshardingWithFilter() throws Exception { { HollowObjectTypeReadState objectTypeReadState = populateTypeStateWithFilter(numRecords); assertDataUnchanged(objectTypeReadState, numRecords); + HollowTypeReshardingStrategy reshardingStrategy = HollowTypeReshardingStrategy.getInstance(objectTypeReadState); // Splitting shards { int prevShardCount = objectTypeReadState.numShards(); int newShardCount = shardingFactor * prevShardCount; - RESHARDING_STRATEGY.reshard(objectTypeReadState, newShardCount); + reshardingStrategy.reshard(objectTypeReadState, objectTypeReadState.numShards(), newShardCount); assertEquals(newShardCount, objectTypeReadState.numShards()); assertEquals(newShardCount, shardingFactor * prevShardCount); @@ -125,7 +68,7 @@ public void testReshardingWithFilter() throws Exception { { int prevShardCount = objectTypeReadState.numShards(); int newShardCount = prevShardCount / shardingFactor; - RESHARDING_STRATEGY.reshard(objectTypeReadState, newShardCount); + reshardingStrategy.reshard(objectTypeReadState, objectTypeReadState.numShards(), newShardCount); assertEquals(newShardCount, objectTypeReadState.numShards()); assertEquals(shardingFactor * newShardCount, prevShardCount); @@ -134,87 +77,4 @@ public void testReshardingWithFilter() throws Exception { } } } - - @Test - public void testReshardingIntermediateStages_expandWithOriginalDataElements() throws Exception { - for (int shardingFactor : new int[]{2, 4}) { - for(int numRecords=1;numRecords<=100000;numRecords+=new Random().nextInt(5000)) - { - HollowObjectTypeReadState expectedTypeState = populateTypeStateWith(numRecords); - - HollowObjectTypeShardsHolder original = expectedTypeState.shardsVolatile; - HollowObjectTypeReadState actualTypeState = new HollowObjectTypeReadState(readStateEngine, MemoryMode.ON_HEAP, schema, schema); - actualTypeState.updateShardsVolatile(RESHARDING_STRATEGY.expandWithOriginalDataElements(original, shardingFactor)); - - assertEquals(shardingFactor * expectedTypeState.numShards(), actualTypeState.numShards()); - assertDataUnchanged(actualTypeState, numRecords); - } - } - } - - @Test - public void testReshardingIntermediateStages_splitDataElementsForOneShard() throws Exception { - for (int shardingFactor : new int[]{2, 4}) { - for(int numRecords=1;numRecords<=100000;numRecords+=new Random().nextInt(5000)) - { - HollowObjectTypeReadState typeState = populateTypeStateWith(numRecords); - - HollowObjectTypeShardsHolder originalShardsHolder = typeState.shardsVolatile; - int originalNumShards = typeState.numShards(); - - // expand shards - typeState.updateShardsVolatile(RESHARDING_STRATEGY.expandWithOriginalDataElements(originalShardsHolder, shardingFactor)); - - for(int i=0; i invocation) { - try { - invocation.get(); - Assert.fail(); - } catch (IllegalStateException e) { - // expected - } - } } diff --git a/hollow/src/test/java/com/netflix/hollow/core/write/HollowTypeWriteStateTest.java b/hollow/src/test/java/com/netflix/hollow/core/write/HollowTypeWriteStateTest.java index 230e6872b5..5590beace4 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/write/HollowTypeWriteStateTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/write/HollowTypeWriteStateTest.java @@ -17,7 +17,6 @@ public class HollowTypeWriteStateTest { - // SNAP: TODO: already has collections @Test public void testReverseDeltaNumShardsWhenNewTypes() { InMemoryBlobStore blobStore = new InMemoryBlobStore();