Lower scale, assert on record count

gchq · Apr 26, 2024 · 1a012af · 1a012af
1 parent 6888938
commit 1a012af
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 8 deletions.
diff --git a/...-test/system-test-suite/src/test/java/sleeper/systemtest/suite/ParallelCompactionsIT.java b/...-test/system-test-suite/src/test/java/sleeper/systemtest/suite/ParallelCompactionsIT.java
@@ -20,6 +20,7 @@
 
 import sleeper.compaction.strategy.impl.BasicCompactionStrategy;
 import sleeper.configuration.properties.validation.IngestFileWritingStrategy;
+import sleeper.core.statestore.FileReference;
 import sleeper.core.util.PollWithRetries;
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.suite.testutil.Expensive;
@@ -38,7 +39,7 @@
 import static sleeper.systemtest.configuration.SystemTestProperty.NUMBER_OF_RECORDS_PER_INGEST;
 import static sleeper.systemtest.configuration.SystemTestProperty.NUMBER_OF_WRITERS;
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.PARALLEL_COMPACTIONS;
-import static sleeper.systemtest.suite.testutil.PartitionsTestHelper.create8192StringPartitions;
+import static sleeper.systemtest.suite.testutil.PartitionsTestHelper.create2048StringPartitions;
 
 @SystemTest
 @Expensive
@@ -52,7 +53,7 @@ void setUp(SleeperSystemTest sleeper) throws Exception {
     @Test
     void shouldApplyOneCompactionPerPartition(SleeperSystemTest sleeper) {
         // Given we configure to compact many partitions
-        sleeper.partitioning().setPartitions(create8192StringPartitions(sleeper));
+        sleeper.partitioning().setPartitions(create2048StringPartitions(sleeper));
         sleeper.updateTableProperties(Map.of(
                 COMPACTION_STRATEGY_CLASS, BasicCompactionStrategy.class.getName(),
                 COMPACTION_FILES_BATCH_SIZE, "10",
@@ -71,7 +72,7 @@ void shouldApplyOneCompactionPerPartition(SleeperSystemTest sleeper) {
         // When we run compaction
         sleeper.compaction()
                 .forceStartTasks(300)
-                .createJobs(8192,
+                .createJobs(2048,
                         PollWithRetries.intervalAndPollingTimeout(
                                 Duration.ofSeconds(10), Duration.ofMinutes(5)))
                 .waitForJobs(
@@ -80,13 +81,15 @@ void shouldApplyOneCompactionPerPartition(SleeperSystemTest sleeper) {
 
         // Then we have one file per partition
         assertThat(sleeper.tableFiles().references())
+                .hasSize(2048)
+                .satisfies(files -> assertThat(files.stream().mapToLong(FileReference::getNumberOfRecords).sum())
+                        .isEqualTo(10_000_000))
                 .allMatch(file -> file.onlyContainsDataForThisPartition() && !file.isCountApproximate(),
                         "only contains data for one partition")
                 .allMatch(file -> file.getJobId() != null,
                         "not assigned to any job")
-                .allMatch(file -> file.getNumberOfRecords() > 1_000 && file.getNumberOfRecords() < 1_300,
-                        "contains an even distribution of records for the partition")
-                .hasSize(8192);
+                .allMatch(file -> file.getNumberOfRecords() > 400 && file.getNumberOfRecords() < 550,
+                        "contains an even distribution of records for the partition");
     }
 
 }
diff --git a/...stem-test-suite/src/test/java/sleeper/systemtest/suite/testutil/PartitionsTestHelper.java b/...stem-test-suite/src/test/java/sleeper/systemtest/suite/testutil/PartitionsTestHelper.java
@@ -42,8 +42,8 @@ public static PartitionTree create512StringPartitions(SleeperSystemTest sleeper)
         return createStringPartitionsFromSplitPointsDirectory(sleeper, "512-partitions.txt");
     }
 
-    public static PartitionTree create8192StringPartitions(SleeperSystemTest sleeper) {
-        return createStringPartitionsFromSplitPointsDirectory(sleeper, "8192-partitions.txt");
+    public static PartitionTree create2048StringPartitions(SleeperSystemTest sleeper) {
+        return createStringPartitionsFromSplitPointsDirectory(sleeper, "2048-partitions.txt");
     }
 
     public static PartitionTree createStringPartitionsFromSplitPointsDirectory(