From c11b404a61c1ca8c0cb02c67486ed6b18f369ba3 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 12 Aug 2024 09:59:39 +0000 Subject: [PATCH 01/49] Set up StateStoreCommitterST --- .../SystemTestStateStoreFakeCommits.java | 11 +++- .../suite/StateStoreCommitterST.java | 55 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index 02e8cec7ab..b1e375d330 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -18,6 +18,7 @@ import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Stream; @@ -25,6 +26,7 @@ public class SystemTestStateStoreFakeCommits { private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; + private final AtomicInteger commitsSent = new AtomicInteger(); public SystemTestStateStoreFakeCommits(SystemTestContext context) { instance = context.instance(); @@ -32,15 +34,20 @@ public SystemTestStateStoreFakeCommits(SystemTestContext context) { } public SystemTestStateStoreFakeCommits sendBatched(Function> buildCommits) { - driver.sendCommitMessages(buildCommits.apply(messageFactory())); + send(buildCommits.apply(messageFactory())); return this; } public SystemTestStateStoreFakeCommits send(Function buildCommit) { - driver.sendCommitMessages(Stream.of(buildCommit.apply(messageFactory()))); + send(Stream.of(buildCommit.apply(messageFactory()))); return this; } + private void send(Stream messages) { + driver.sendCommitMessages(messages + .peek(message -> commitsSent.incrementAndGet())); + } + private StateStoreCommitMessageFactory messageFactory() { return new StateStoreCommitMessageFactory(instance.getTableStatus().getTableUniqueId()); } diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java new file mode 100644 index 0000000000..b866089df4 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -0,0 +1,55 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.suite; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import sleeper.core.partition.PartitionTree; +import sleeper.core.partition.PartitionsBuilder; +import sleeper.core.statestore.FileReferenceFactory; +import sleeper.systemtest.dsl.SleeperSystemTest; +import sleeper.systemtest.suite.testutil.SystemTest; + +import java.util.stream.IntStream; + +import static sleeper.systemtest.suite.fixtures.SystemTestInstance.MAIN; +import static sleeper.systemtest.suite.fixtures.SystemTestSchema.DEFAULT_SCHEMA; + +@SystemTest +public class StateStoreCommitterST { + + @BeforeEach + void setUp(SleeperSystemTest sleeper) { + sleeper.connectToInstance(MAIN); + } + + @Test + void shouldAddManyFiles(SleeperSystemTest sleeper) { + // Given + PartitionTree partitions = new PartitionsBuilder(DEFAULT_SCHEMA).singlePartition("root").buildTree(); + sleeper.partitioning().setPartitions(partitions); + + // When + FileReferenceFactory fileFactory = FileReferenceFactory.from(partitions); + sleeper.stateStore().fakeCommits().sendBatched( + commitFactory -> IntStream.rangeClosed(1, 1000) + .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) + .map(commitFactory::addFile)); + + // Then + } +} From 3a274096ba39a6bb2d40f246a0efc5343fa8e79c Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:12:51 +0000 Subject: [PATCH 02/49] Refactor StateStoreCommitRequest to hold table ID --- .../commit/StateStoreCommitRequest.java | 32 ++++++++++++++---- .../sleeper/commit/StateStoreCommitter.java | 33 ++++++------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java index 4bf42dc004..f288ebbd9c 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java @@ -17,6 +17,7 @@ import sleeper.compaction.job.commit.CompactionJobCommitRequest; import sleeper.compaction.job.commit.CompactionJobIdAssignmentCommitRequest; +import sleeper.core.statestore.StateStoreException; import sleeper.core.statestore.commit.SplitPartitionCommitRequest; import sleeper.core.statestore.commit.StateStoreCommitRequestInS3; import sleeper.ingest.job.commit.IngestAddFilesCommitRequest; @@ -29,6 +30,8 @@ public class StateStoreCommitRequest { private final Object request; + private final String tableId; + private final ApplyRequest applyRequest; /** * Creates a request to commit the results of a compaction job. @@ -37,7 +40,7 @@ public class StateStoreCommitRequest { * @return a state store commit request */ public static StateStoreCommitRequest forCompactionJob(CompactionJobCommitRequest request) { - return new StateStoreCommitRequest(request); + return new StateStoreCommitRequest(request, request.getJob().getTableId(), committer -> committer.apply(request)); } /** @@ -47,7 +50,7 @@ public static StateStoreCommitRequest forCompactionJob(CompactionJobCommitReques * @return a state store commit request */ public static StateStoreCommitRequest forCompactionJobIdAssignment(CompactionJobIdAssignmentCommitRequest request) { - return new StateStoreCommitRequest(request); + return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); } /** @@ -57,7 +60,7 @@ public static StateStoreCommitRequest forCompactionJobIdAssignment(CompactionJob * @return a state store commit request */ public static StateStoreCommitRequest forIngestAddFiles(IngestAddFilesCommitRequest request) { - return new StateStoreCommitRequest(request); + return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); } /** @@ -67,7 +70,7 @@ public static StateStoreCommitRequest forIngestAddFiles(IngestAddFilesCommitRequ * @return a state store commit request */ public static StateStoreCommitRequest forSplitPartition(SplitPartitionCommitRequest request) { - return new StateStoreCommitRequest(request); + return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); } /** @@ -77,17 +80,27 @@ public static StateStoreCommitRequest forSplitPartition(SplitPartitionCommitRequ * @return a state store commit request */ public static StateStoreCommitRequest storedInS3(StateStoreCommitRequestInS3 request) { - return new StateStoreCommitRequest(request); + return new StateStoreCommitRequest(request, null, committer -> committer.apply(request)); } - private StateStoreCommitRequest(Object request) { + private StateStoreCommitRequest(Object request, String tableId, ApplyRequest applyRequest) { this.request = request; + this.tableId = tableId; + this.applyRequest = applyRequest; } public Object getRequest() { return request; } + public String getTableId() { + return tableId; + } + + void apply(StateStoreCommitter committer) throws StateStoreException { + applyRequest.apply(committer); + } + @Override public int hashCode() { return Objects.hash(request); @@ -110,4 +123,11 @@ public String toString() { return "StateStoreCommitRequest{request=" + request + "}"; } + /** + * Applies the current request with a given committer. + */ + @FunctionalInterface + private interface ApplyRequest { + void apply(StateStoreCommitter committer) throws StateStoreException; + } } diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java index 4d26c19612..a0b8874506 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java @@ -92,23 +92,10 @@ public void applyFromJson(String json) throws StateStoreException { * @param request the commit request */ public void apply(StateStoreCommitRequest request) throws StateStoreException { - Object requestObj = request.getRequest(); - if (requestObj instanceof CompactionJobCommitRequest) { - apply((CompactionJobCommitRequest) requestObj); - } else if (requestObj instanceof IngestAddFilesCommitRequest) { - apply((IngestAddFilesCommitRequest) requestObj); - } else if (requestObj instanceof StateStoreCommitRequestInS3) { - apply((StateStoreCommitRequestInS3) requestObj); - } else if (requestObj instanceof CompactionJobIdAssignmentCommitRequest) { - apply((CompactionJobIdAssignmentCommitRequest) requestObj); - } else if (requestObj instanceof SplitPartitionCommitRequest) { - apply((SplitPartitionCommitRequest) requestObj); - } else { - throw new IllegalArgumentException("Unsupported commit request type: " + requestObj.getClass().getName()); - } + request.apply(this); } - private void apply(CompactionJobCommitRequest request) throws StateStoreException { + void apply(CompactionJobCommitRequest request) throws StateStoreException { CompactionJob job = request.getJob(); try { CompactionJobCommitter.updateStateStoreSuccess(job, request.getRecordsWritten(), @@ -133,7 +120,7 @@ private void apply(CompactionJobCommitRequest request) throws StateStoreExceptio } } - private void apply(IngestAddFilesCommitRequest request) throws StateStoreException { + void apply(IngestAddFilesCommitRequest request) throws StateStoreException { StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); List files = AllReferencesToAFile.newFilesWithReferences(request.getFileReferences()); stateStore.addFilesWithReferences(files); @@ -147,12 +134,12 @@ private void apply(IngestAddFilesCommitRequest request) throws StateStoreExcepti } } - private void apply(SplitPartitionCommitRequest request) throws StateStoreException { + void apply(SplitPartitionCommitRequest request) throws StateStoreException { StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); stateStore.atomicallyUpdatePartitionAndCreateNewOnes(request.getParentPartition(), request.getLeftChild(), request.getRightChild()); } - private void apply(StateStoreCommitRequestInS3 request) throws StateStoreException { + void apply(StateStoreCommitRequestInS3 request) throws StateStoreException { String json = loadFromDataBucket.loadFromDataBucket(request.getKeyInS3()); StateStoreCommitRequest requestFromS3 = deserialiser.fromJson(json); if (requestFromS3.getRequest() instanceof StateStoreCommitRequestInS3) { @@ -161,6 +148,11 @@ private void apply(StateStoreCommitRequestInS3 request) throws StateStoreExcepti apply(requestFromS3); } + void apply(CompactionJobIdAssignmentCommitRequest request) throws StateStoreException { + StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); + stateStore.assignJobIds(request.getAssignJobIdRequests()); + } + /** * Loads S3 objects from the data bucket. */ @@ -173,9 +165,4 @@ public interface LoadS3ObjectFromDataBucket { */ String loadFromDataBucket(String key); } - - private void apply(CompactionJobIdAssignmentCommitRequest request) throws StateStoreException { - StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); - stateStore.assignJobIds(request.getAssignJobIdRequests()); - } } From e8175a75b9787464bf50bb8fbcaafd08b734f395 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:46:44 +0000 Subject: [PATCH 03/49] Move reading from S3 into StateStoreCommitRequestDeserialiser --- .../commit/StateStoreCommitRequest.java | 11 ----- .../StateStoreCommitRequestDeserialiser.java | 44 ++++++++++++++++--- .../sleeper/commit/StateStoreCommitter.java | 14 +----- ...ateStoreCommitRequestDeserialiserTest.java | 32 +++++++++++++- .../commit/StateStoreCommitterTest.java | 41 ----------------- .../AwsStateStoreCommitterDriverIT.java | 6 ++- 6 files changed, 74 insertions(+), 74 deletions(-) diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java index f288ebbd9c..481134a0d2 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java @@ -19,7 +19,6 @@ import sleeper.compaction.job.commit.CompactionJobIdAssignmentCommitRequest; import sleeper.core.statestore.StateStoreException; import sleeper.core.statestore.commit.SplitPartitionCommitRequest; -import sleeper.core.statestore.commit.StateStoreCommitRequestInS3; import sleeper.ingest.job.commit.IngestAddFilesCommitRequest; import java.util.Objects; @@ -73,16 +72,6 @@ public static StateStoreCommitRequest forSplitPartition(SplitPartitionCommitRequ return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); } - /** - * Creates a request which is stored in S3. - * - * @param request the commit request - * @return a state store commit request - */ - public static StateStoreCommitRequest storedInS3(StateStoreCommitRequestInS3 request) { - return new StateStoreCommitRequest(request, null, committer -> committer.apply(request)); - } - private StateStoreCommitRequest(Object request, String tableId, ApplyRequest applyRequest) { this.request = request; this.tableId = tableId; diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java index b4b6c94643..b1df9f1cd7 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sleeper.commit.StateStoreCommitter.LoadS3ObjectFromDataBucket; import sleeper.compaction.job.CompactionJob; import sleeper.compaction.job.CompactionJobJsonSerDe; import sleeper.compaction.job.commit.CompactionJobCommitRequest; @@ -46,11 +47,20 @@ public class StateStoreCommitRequestDeserialiser { public static final Logger LOGGER = LoggerFactory.getLogger(StateStoreCommitRequestDeserialiser.class); private final Gson gson; + private final Gson gsonFromDataBucket; + private final LoadS3ObjectFromDataBucket loadFromDataBucket; + + public StateStoreCommitRequestDeserialiser( + TablePropertiesProvider tablePropertiesProvider, LoadS3ObjectFromDataBucket loadFromDataBucket) { + gson = gson(tablePropertiesProvider, this::fromDataBucket); + gsonFromDataBucket = gson(tablePropertiesProvider, DeserialiseFromDataBucket.refuseWhileReadingFromBucket()); + this.loadFromDataBucket = loadFromDataBucket; + } - public StateStoreCommitRequestDeserialiser(TablePropertiesProvider tablePropertiesProvider) { - gson = GsonConfig.standardBuilder() + private static Gson gson(TablePropertiesProvider tablePropertiesProvider, DeserialiseFromDataBucket readFromDataBucket) { + return GsonConfig.standardBuilder() .registerTypeAdapter(CompactionJob.class, new CompactionJobJsonSerDe()) - .registerTypeAdapter(StateStoreCommitRequest.class, new WrapperDeserialiser()) + .registerTypeAdapter(StateStoreCommitRequest.class, new WrapperDeserialiser(readFromDataBucket)) .registerTypeAdapter(SplitPartitionCommitRequest.class, new SplitPartitionDeserialiser(tablePropertiesProvider)) .serializeNulls() .create(); @@ -66,11 +76,22 @@ public StateStoreCommitRequest fromJson(String jsonString) { return gson.fromJson(jsonString, StateStoreCommitRequest.class); } + private StateStoreCommitRequest fromDataBucket(StateStoreCommitRequestInS3 request) { + String json = loadFromDataBucket.loadFromDataBucket(request.getKeyInS3()); + return gsonFromDataBucket.fromJson(json, StateStoreCommitRequest.class); + } + /** * Deserialises the commit request by reading the type. */ private static class WrapperDeserialiser implements JsonDeserializer { + private final DeserialiseFromDataBucket fromDataBucket; + + private WrapperDeserialiser(DeserialiseFromDataBucket fromDataBucket) { + this.fromDataBucket = fromDataBucket; + } + @Override public StateStoreCommitRequest deserialize(JsonElement json, Type wrapperType, JsonDeserializationContext context) throws JsonParseException { @@ -88,15 +109,15 @@ public StateStoreCommitRequest deserialize(JsonElement json, Type wrapperType, J case INGEST_ADD_FILES: return StateStoreCommitRequest.forIngestAddFiles( context.deserialize(requestObj, IngestAddFilesCommitRequest.class)); - case STORED_IN_S3: - return StateStoreCommitRequest.storedInS3( - context.deserialize(requestObj, StateStoreCommitRequestInS3.class)); case COMPACTION_JOB_ID_ASSIGNMENT: return StateStoreCommitRequest.forCompactionJobIdAssignment( context.deserialize(requestObj, CompactionJobIdAssignmentCommitRequest.class)); case SPLIT_PARTITION: return StateStoreCommitRequest.forSplitPartition( context.deserialize(requestObj, SplitPartitionCommitRequest.class)); + case STORED_IN_S3: + return fromDataBucket.read( + context.deserialize(requestObj, StateStoreCommitRequestInS3.class)); default: throw new CommitRequestValidationException("Unrecognised request type"); } @@ -138,4 +159,15 @@ public SplitPartitionCommitRequest deserialize(JsonElement jsonElement, Type typ return new SplitPartitionCommitRequest(tableId, parentPartition, leftChildPartition, rightChildPartition); } } + + @FunctionalInterface + private interface DeserialiseFromDataBucket { + StateStoreCommitRequest read(StateStoreCommitRequestInS3 request); + + static DeserialiseFromDataBucket refuseWhileReadingFromBucket() { + return request -> { + throw new IllegalArgumentException("Found a request stored in S3 pointing to another S3 object: " + request.getKeyInS3()); + }; + } + } } diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java index a0b8874506..601e1cfa4b 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java @@ -30,7 +30,6 @@ import sleeper.core.statestore.StateStore; import sleeper.core.statestore.StateStoreException; import sleeper.core.statestore.commit.SplitPartitionCommitRequest; -import sleeper.core.statestore.commit.StateStoreCommitRequestInS3; import sleeper.core.statestore.exception.FileAlreadyExistsException; import sleeper.core.statestore.exception.FileNotFoundException; import sleeper.core.statestore.exception.FileReferenceNotAssignedToJobException; @@ -59,7 +58,6 @@ public class StateStoreCommitter { private final CompactionJobStatusStore compactionJobStatusStore; private final IngestJobStatusStore ingestJobStatusStore; private final GetStateStoreByTableId stateStoreProvider; - private final LoadS3ObjectFromDataBucket loadFromDataBucket; private final Supplier timeSupplier; public StateStoreCommitter( @@ -69,11 +67,10 @@ public StateStoreCommitter( GetStateStoreByTableId stateStoreProvider, LoadS3ObjectFromDataBucket loadFromDataBucket, Supplier timeSupplier) { - this.deserialiser = new StateStoreCommitRequestDeserialiser(tablePropertiesProvider); + this.deserialiser = new StateStoreCommitRequestDeserialiser(tablePropertiesProvider, loadFromDataBucket); this.compactionJobStatusStore = compactionJobStatusStore; this.ingestJobStatusStore = ingestJobStatusStore; this.stateStoreProvider = stateStoreProvider; - this.loadFromDataBucket = loadFromDataBucket; this.timeSupplier = timeSupplier; } @@ -139,15 +136,6 @@ void apply(SplitPartitionCommitRequest request) throws StateStoreException { stateStore.atomicallyUpdatePartitionAndCreateNewOnes(request.getParentPartition(), request.getLeftChild(), request.getRightChild()); } - void apply(StateStoreCommitRequestInS3 request) throws StateStoreException { - String json = loadFromDataBucket.loadFromDataBucket(request.getKeyInS3()); - StateStoreCommitRequest requestFromS3 = deserialiser.fromJson(json); - if (requestFromS3.getRequest() instanceof StateStoreCommitRequestInS3) { - throw new IllegalArgumentException("Found a request stored in S3 pointing to another S3 object: " + request.getKeyInS3()); - } - apply(requestFromS3); - } - void apply(CompactionJobIdAssignmentCommitRequest request) throws StateStoreException { StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); stateStore.assignJobIds(request.getAssignJobIdRequests()); diff --git a/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java b/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java index eadb5a7a77..89dbe59c25 100644 --- a/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java +++ b/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java @@ -43,7 +43,9 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -56,6 +58,7 @@ public class StateStoreCommitRequestDeserialiserTest { private final InstanceProperties instanceProperties = createTestInstanceProperties(); private final List tables = new ArrayList<>(); + private final Map dataBucketObjectByKey = new HashMap<>(); @Test void shouldDeserialiseCompactionJobCommitRequest() { @@ -189,13 +192,38 @@ void shouldDeserialiseSplitPartitionCommitRequest() { @Test void shouldDeserialiseCommitRequestInS3() { // Given + FileReference file = FileReference.builder() + .filename("file.parquet") + .partitionId("root") + .numberOfRecords(100L) + .onlyContainsDataForThisPartition(true) + .build(); + IngestAddFilesCommitRequest requestInBucket = IngestAddFilesCommitRequest.builder() + .tableId("test-table") + .fileReferences(List.of(file)) + .build(); String s3Key = StateStoreCommitRequestInS3.createFileS3Key("test-table", "test-file"); + dataBucketObjectByKey.put(s3Key, new IngestAddFilesCommitRequestSerDe().toJson(requestInBucket)); StateStoreCommitRequestInS3 commitRequest = new StateStoreCommitRequestInS3(s3Key); String jsonString = new StateStoreCommitRequestInS3SerDe().toJson(commitRequest); // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.storedInS3(commitRequest)); + .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(requestInBucket)); + } + + @Test + void shouldRefuseReferenceToS3HeldInS3() throws Exception { + // Given we have a request pointing to itself + String s3Key = StateStoreCommitRequestInS3.createFileS3Key("test-table", "test-file"); + StateStoreCommitRequestInS3 commitRequest = new StateStoreCommitRequestInS3(s3Key); + dataBucketObjectByKey.put(s3Key, new StateStoreCommitRequestInS3SerDe().toJson(commitRequest)); + String jsonString = new StateStoreCommitRequestInS3SerDe().toJson(commitRequest); + + // When / Then + StateStoreCommitRequestDeserialiser deserialiser = deserialiser(); + assertThatThrownBy(() -> deserialiser.fromJson(jsonString)) + .isInstanceOf(IllegalArgumentException.class); } @Test @@ -215,6 +243,6 @@ private void createTable(String tableId, Schema schema) { } private StateStoreCommitRequestDeserialiser deserialiser() { - return new StateStoreCommitRequestDeserialiser(new FixedTablePropertiesProvider(tables)); + return new StateStoreCommitRequestDeserialiser(new FixedTablePropertiesProvider(tables), dataBucketObjectByKey::get); } } diff --git a/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitterTest.java b/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitterTest.java index 0c3fe903a8..c96e12b989 100644 --- a/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitterTest.java +++ b/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitterTest.java @@ -38,14 +38,11 @@ import sleeper.core.statestore.StateStore; import sleeper.core.statestore.StateStoreException; import sleeper.core.statestore.commit.SplitPartitionCommitRequest; -import sleeper.core.statestore.commit.StateStoreCommitRequestInS3; -import sleeper.core.statestore.commit.StateStoreCommitRequestInS3SerDe; import sleeper.core.statestore.exception.FileReferenceAssignedToJobException; import sleeper.core.statestore.exception.FileReferenceNotFoundException; import sleeper.core.statestore.exception.ReplaceRequestsFailedException; import sleeper.ingest.job.IngestJob; import sleeper.ingest.job.commit.IngestAddFilesCommitRequest; -import sleeper.ingest.job.commit.IngestAddFilesCommitRequestSerDe; import sleeper.ingest.job.status.InMemoryIngestJobStatusStore; import java.time.Duration; @@ -312,44 +309,6 @@ void shouldApplyIngestStreamAddFilesCommitRequest() throws Exception { } } - @Nested - @DisplayName("Read request from S3") - class ReadFromS3 { - - @Test - void shouldApplyIngestStreamAddFilesCommitRequestHeldInS3() throws Exception { - // Given - StateStore stateStore = createTable("test-table"); - FileReference outputFile = fileFactory.rootFile("output.parquet", 123L); - IngestAddFilesCommitRequest commitRequest = IngestAddFilesCommitRequest.builder() - .tableId("test-table") - .fileReferences(List.of(outputFile)) - .build(); - String s3Key = StateStoreCommitRequestInS3.createFileS3Key("test-table", "test-file"); - dataBucketObjectByKey.put(s3Key, new IngestAddFilesCommitRequestSerDe().toJson(commitRequest)); - - // When - committer().apply(StateStoreCommitRequest.storedInS3(new StateStoreCommitRequestInS3(s3Key))); - - // Then - assertThat(stateStore.getFileReferences()).containsExactly(outputFile); - assertThat(ingestJobStatusStore.getAllJobs("test-table")).isEmpty(); - } - - @Test - void shouldRefuseReferenceToS3HeldInS3() throws Exception { - // Given we have a request pointing to itself - String s3Key = StateStoreCommitRequestInS3.createFileS3Key("test-table", "test-file"); - StateStoreCommitRequestInS3 request = new StateStoreCommitRequestInS3(s3Key); - dataBucketObjectByKey.put(s3Key, new StateStoreCommitRequestInS3SerDe().toJson(request)); - - // When / Then - StateStoreCommitter committer = committer(); - assertThatThrownBy(() -> committer.apply(StateStoreCommitRequest.storedInS3(request))) - .isInstanceOf(IllegalArgumentException.class); - } - } - @Nested @DisplayName("Split a partition to create child partitions") class SplitPartition { diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java index 87db59cbf4..6b8f5f393b 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java @@ -23,6 +23,7 @@ import sleeper.commit.StateStoreCommitRequest; import sleeper.commit.StateStoreCommitRequestDeserialiser; +import sleeper.commit.StateStoreCommitter.LoadS3ObjectFromDataBucket; import sleeper.core.partition.PartitionTree; import sleeper.core.partition.PartitionsBuilder; import sleeper.core.statestore.FileReference; @@ -129,7 +130,10 @@ private String getMessageGroupId(Message message) { } private StateStoreCommitRequest readCommitRequest(Message message) { - return new StateStoreCommitRequestDeserialiser(instance.getTablePropertiesProvider()) + LoadS3ObjectFromDataBucket noLoadFromS3 = key -> { + throw new UnsupportedOperationException("Did not expect message held in S3"); + }; + return new StateStoreCommitRequestDeserialiser(instance.getTablePropertiesProvider(), noLoadFromS3) .fromJson(message.getBody()); } From b9df6d6918880d774b6dc260d1dbc74e20425166 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:48:18 +0000 Subject: [PATCH 04/49] Assert on StateStoreCommitRequest.getTableId --- ...tateStoreCommitRequestDeserialiserTest.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java b/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java index 89dbe59c25..fb2183236b 100644 --- a/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java +++ b/java/statestore-commit/src/test/java/sleeper/commit/StateStoreCommitRequestDeserialiserTest.java @@ -79,7 +79,8 @@ void shouldDeserialiseCompactionJobCommitRequest() { // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.forCompactionJob(compactionJobCommitRequest)); + .isEqualTo(StateStoreCommitRequest.forCompactionJob(compactionJobCommitRequest)) + .extracting(StateStoreCommitRequest::getTableId).isEqualTo("test-table"); } @Test @@ -105,7 +106,8 @@ void shouldDeserialiseCompactionJobIdAssignmentCommitRequest() { // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.forCompactionJobIdAssignment(jobIdAssignmentRequest)); + .isEqualTo(StateStoreCommitRequest.forCompactionJobIdAssignment(jobIdAssignmentRequest)) + .extracting(StateStoreCommitRequest::getTableId).isEqualTo("test-table"); } @Test @@ -140,7 +142,8 @@ void shouldDeserialiseIngestJobCommitRequest() { // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(ingestJobCommitRequest)); + .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(ingestJobCommitRequest)) + .extracting(StateStoreCommitRequest::getTableId).isEqualTo("test-table-id"); } @Test @@ -166,7 +169,8 @@ void shouldDeserialiseIngestCommitRequestWithNoJob() { // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(ingestJobCommitRequest)); + .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(ingestJobCommitRequest)) + .extracting(StateStoreCommitRequest::getTableId).isEqualTo("test-table"); } @Test @@ -186,7 +190,8 @@ void shouldDeserialiseSplitPartitionCommitRequest() { // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.forSplitPartition(splitPartitionCommitRequest)); + .isEqualTo(StateStoreCommitRequest.forSplitPartition(splitPartitionCommitRequest)) + .extracting(StateStoreCommitRequest::getTableId).isEqualTo("test-table"); } @Test @@ -209,7 +214,8 @@ void shouldDeserialiseCommitRequestInS3() { // When / Then assertThat(deserialiser().fromJson(jsonString)) - .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(requestInBucket)); + .isEqualTo(StateStoreCommitRequest.forIngestAddFiles(requestInBucket)) + .extracting(StateStoreCommitRequest::getTableId).isEqualTo("test-table"); } @Test From 409c4260f6c1b6125695cdc8ad6d00bdf54cc2c4 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:06:32 +0000 Subject: [PATCH 05/49] Add missing Javadoc --- .../sleeper/commit/StateStoreCommitRequestDeserialiser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java index b1df9f1cd7..155159cd51 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequestDeserialiser.java @@ -160,6 +160,10 @@ public SplitPartitionCommitRequest deserialize(JsonElement jsonElement, Type typ } } + /** + * Reads and deserialises a commit request from the data bucket. An alternative implementation will refuse reading + * from the bucket because the pointer was already stored in S3. + */ @FunctionalInterface private interface DeserialiseFromDataBucket { StateStoreCommitRequest read(StateStoreCommitRequestInS3 request); From a2a07065d253641af9b14d914910a31c4a06ff2e Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:22:41 +0000 Subject: [PATCH 06/49] Rename methods to apply commit requests --- .../main/java/sleeper/commit/StateStoreCommitRequest.java | 8 ++++---- .../src/main/java/sleeper/commit/StateStoreCommitter.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java index 481134a0d2..91e65a51a6 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitRequest.java @@ -39,7 +39,7 @@ public class StateStoreCommitRequest { * @return a state store commit request */ public static StateStoreCommitRequest forCompactionJob(CompactionJobCommitRequest request) { - return new StateStoreCommitRequest(request, request.getJob().getTableId(), committer -> committer.apply(request)); + return new StateStoreCommitRequest(request, request.getJob().getTableId(), committer -> committer.commitCompaction(request)); } /** @@ -49,7 +49,7 @@ public static StateStoreCommitRequest forCompactionJob(CompactionJobCommitReques * @return a state store commit request */ public static StateStoreCommitRequest forCompactionJobIdAssignment(CompactionJobIdAssignmentCommitRequest request) { - return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); + return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.assignCompactionInputFiles(request)); } /** @@ -59,7 +59,7 @@ public static StateStoreCommitRequest forCompactionJobIdAssignment(CompactionJob * @return a state store commit request */ public static StateStoreCommitRequest forIngestAddFiles(IngestAddFilesCommitRequest request) { - return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); + return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.addFiles(request)); } /** @@ -69,7 +69,7 @@ public static StateStoreCommitRequest forIngestAddFiles(IngestAddFilesCommitRequ * @return a state store commit request */ public static StateStoreCommitRequest forSplitPartition(SplitPartitionCommitRequest request) { - return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.apply(request)); + return new StateStoreCommitRequest(request, request.getTableId(), committer -> committer.splitPartition(request)); } private StateStoreCommitRequest(Object request, String tableId, ApplyRequest applyRequest) { diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java index 601e1cfa4b..48492c8191 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java @@ -92,7 +92,7 @@ public void apply(StateStoreCommitRequest request) throws StateStoreException { request.apply(this); } - void apply(CompactionJobCommitRequest request) throws StateStoreException { + void commitCompaction(CompactionJobCommitRequest request) throws StateStoreException { CompactionJob job = request.getJob(); try { CompactionJobCommitter.updateStateStoreSuccess(job, request.getRecordsWritten(), @@ -117,7 +117,7 @@ void apply(CompactionJobCommitRequest request) throws StateStoreException { } } - void apply(IngestAddFilesCommitRequest request) throws StateStoreException { + void addFiles(IngestAddFilesCommitRequest request) throws StateStoreException { StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); List files = AllReferencesToAFile.newFilesWithReferences(request.getFileReferences()); stateStore.addFilesWithReferences(files); @@ -131,12 +131,12 @@ void apply(IngestAddFilesCommitRequest request) throws StateStoreException { } } - void apply(SplitPartitionCommitRequest request) throws StateStoreException { + void splitPartition(SplitPartitionCommitRequest request) throws StateStoreException { StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); stateStore.atomicallyUpdatePartitionAndCreateNewOnes(request.getParentPartition(), request.getLeftChild(), request.getRightChild()); } - void apply(CompactionJobIdAssignmentCommitRequest request) throws StateStoreException { + void assignCompactionInputFiles(CompactionJobIdAssignmentCommitRequest request) throws StateStoreException { StateStore stateStore = stateStoreProvider.getByTableId(request.getTableId()); stateStore.assignJobIds(request.getAssignJobIdRequests()); } From 71ee3aa6284f4b4ac2185c64782dc22963491059 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:35:32 +0000 Subject: [PATCH 07/49] Adjust state store committer logs --- .../src/main/java/sleeper/commit/StateStoreCommitter.java | 8 +++++--- .../committer/lambda/StateStoreCommitterLambda.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java index 48492c8191..08a18e021d 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java @@ -90,6 +90,8 @@ public void applyFromJson(String json) throws StateStoreException { */ public void apply(StateStoreCommitRequest request) throws StateStoreException { request.apply(this); + LOGGER.info("Applied request to table ID {} with type {}", + request.getTableId(), request.getRequest().getClass().getSimpleName()); } void commitCompaction(CompactionJobCommitRequest request) throws StateStoreException { @@ -99,7 +101,7 @@ void commitCompaction(CompactionJobCommitRequest request) throws StateStoreExcep stateStoreProvider.getByTableId(job.getTableId())); compactionJobStatusStore.jobCommitted(compactionJobCommitted(job, timeSupplier.get()) .taskId(request.getTaskId()).jobRunId(request.getJobRunId()).build()); - LOGGER.info("Successfully committed compaction job {} to table with ID {}", job.getId(), job.getTableId()); + LOGGER.debug("Successfully committed compaction job {}", job.getId()); } catch (ReplaceRequestsFailedException e) { Exception failure = e.getFailures().get(0); if (failure instanceof FileNotFoundException @@ -125,9 +127,9 @@ void addFiles(IngestAddFilesCommitRequest request) throws StateStoreException { if (job != null) { ingestJobStatusStore.jobAddedFiles(IngestJobAddedFilesEvent.ingestJobAddedFiles(job, files, request.getWrittenTime()) .taskId(request.getTaskId()).jobRunId(request.getJobRunId()).build()); - LOGGER.info("Successfully committed new files for ingest job {} to table with ID {}", job.getId(), request.getTableId()); + LOGGER.debug("Successfully committed new files for ingest job {}", job.getId()); } else { - LOGGER.info("Successfully committed new files for ingest to table with ID {}", request.getTableId()); + LOGGER.debug("Successfully committed new files for ingest with no job"); } } diff --git a/java/statestore-lambda/src/main/java/sleeper/statestore/committer/lambda/StateStoreCommitterLambda.java b/java/statestore-lambda/src/main/java/sleeper/statestore/committer/lambda/StateStoreCommitterLambda.java index 2eee39ca5a..ebb9d03e2d 100644 --- a/java/statestore-lambda/src/main/java/sleeper/statestore/committer/lambda/StateStoreCommitterLambda.java +++ b/java/statestore-lambda/src/main/java/sleeper/statestore/committer/lambda/StateStoreCommitterLambda.java @@ -69,7 +69,7 @@ public SQSBatchResponse handleRequest(SQSEvent event, Context context) { .build(); for (int i = 0; i < messages.size(); i++) { SQSMessage message = messages.get(i); - LOGGER.info("Found message: {}", message.getBody()); + LOGGER.debug("Found message: {}", message.getBody()); try { DynamoDBUtils.retryOnThrottlingException(throttlingRetries, () -> { try { From 0c9f094a1d339609ad4eccfee0b61f6ad6215015 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:30:43 +0000 Subject: [PATCH 08/49] Wait for state store commits in-memory --- .../sleeper/commit/StateStoreCommitter.java | 9 ++-- .../AwsStateStoreCommitterDriver.java | 8 +++ .../statestore/StateStoreCommitSummary.java | 42 +++++++++++++++ .../statestore/StateStoreCommitterDriver.java | 3 ++ .../statestore/StateStoreCommitterRun.java | 53 +++++++++++++++++++ .../SystemTestStateStoreFakeCommits.java | 52 ++++++++++++++++-- .../SystemTestStateStoreFakeCommitsTest.java | 11 ++-- .../drivers/InMemoryStateStoreCommitter.java | 36 +++++++++++-- .../suite/StateStoreCommitterST.java | 17 ++++-- 9 files changed, 214 insertions(+), 17 deletions(-) create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java index 08a18e021d..ffcde2a857 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java @@ -77,10 +77,13 @@ public StateStoreCommitter( /** * Applies a state store commit request. * - * @param json the commit request JSON string + * @param json the commit request JSON string + * @return the commit request */ - public void applyFromJson(String json) throws StateStoreException { - apply(deserialiser.fromJson(json)); + public StateStoreCommitRequest applyFromJson(String json) throws StateStoreException { + StateStoreCommitRequest request = deserialiser.fromJson(json); + apply(request); + return request; } /** diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java index 1c421909fc..759ac83be7 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java @@ -23,7 +23,9 @@ import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; +import java.time.Instant; import java.util.List; import java.util.UUID; import java.util.stream.Stream; @@ -46,6 +48,12 @@ public void sendCommitMessages(Stream messages) { SplitIntoBatches.inParallelBatchesOf(10, messages, this::sendMessageBatch); } + @Override + public List getRunsAfter(Instant startTime) { + // TODO Auto-generated method stub + return null; + } + private void sendMessageBatch(List batch) { sqs.sendMessageBatch(new SendMessageBatchRequest() .withQueueUrl(instance.getInstanceProperties().get(STATESTORE_COMMITTER_QUEUE_URL)) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java new file mode 100644 index 0000000000..2ba8810192 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java @@ -0,0 +1,42 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.time.Instant; + +public class StateStoreCommitSummary { + private final String tableId; + private final String type; + private final Instant finishTime; + + public StateStoreCommitSummary(String tableId, String type, Instant finishTime) { + this.tableId = tableId; + this.type = type; + this.finishTime = finishTime; + } + + public String getTableId() { + return tableId; + } + + public String getType() { + return type; + } + + public Instant getFinishTime() { + return finishTime; + } +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java index 435c2fe797..81ad7ee0fa 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java @@ -15,10 +15,13 @@ */ package sleeper.systemtest.dsl.statestore; +import java.time.Instant; +import java.util.List; import java.util.stream.Stream; public interface StateStoreCommitterDriver { void sendCommitMessages(Stream messages); + List getRunsAfter(Instant startTime); } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java new file mode 100644 index 0000000000..27010627d1 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java @@ -0,0 +1,53 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.summingInt; + +public class StateStoreCommitterRun { + + private final Instant startTime; + private final Instant finishTime; + private final List commits; + + public StateStoreCommitterRun(Instant startTime, Instant finishTime, List commits) { + this.startTime = startTime; + this.finishTime = finishTime; + this.commits = commits; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getFinishTime() { + return finishTime; + } + + public List getCommits() { + return commits; + } + + public Map countCommitsByTableId() { + return commits.stream() + .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); + } +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index b1e375d330..d912443c0c 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -15,22 +15,32 @@ */ package sleeper.systemtest.dsl.statestore; +import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; -import java.util.concurrent.atomic.AtomicInteger; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Stream; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.summingInt; + public class SystemTestStateStoreFakeCommits { private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; - private final AtomicInteger commitsSent = new AtomicInteger(); + private final Map waitForNumCommitsByTableId = new ConcurrentHashMap<>(); + private Instant findCommitsFromTime; public SystemTestStateStoreFakeCommits(SystemTestContext context) { instance = context.instance(); driver = context.instance().adminDrivers().stateStoreCommitter(context); + findCommitsFromTime = context.reporting().getRecordingStartTime(); } public SystemTestStateStoreFakeCommits sendBatched(Function> buildCommits) { @@ -43,9 +53,45 @@ public SystemTestStateStoreFakeCommits send(Function { + List runs = driver.getRunsAfter(findCommitsFromTime); + decrementWaitForNumCommits(runs); + updateFindCommitsFromTime(runs); + return waitForNumCommitsByTableId.isEmpty(); + }); + return this; + } + private void send(Stream messages) { driver.sendCommitMessages(messages - .peek(message -> commitsSent.incrementAndGet())); + .peek(message -> waitForNumCommitsByTableId.compute( + message.getTableId(), + (id, count) -> count == null ? 1 : count + 1))); + } + + private void decrementWaitForNumCommits(List runs) { + Map numCommitsByTableId = runs.stream() + .flatMap(run -> run.getCommits().stream()) + .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); + numCommitsByTableId.forEach((tableId, numCommits) -> { + waitForNumCommitsByTableId.compute(tableId, (id, count) -> { + if (count == null) { + return null; + } else if (numCommits >= count) { + return null; + } else { + return count - numCommits; + } + }); + }); + } + + private void updateFindCommitsFromTime(List runs) { + runs.stream() + .map(StateStoreCommitterRun::getFinishTime) + .max(Comparator.naturalOrder()) + .ifPresent(lastTime -> findCommitsFromTime = lastTime); } private StateStoreCommitMessageFactory messageFactory() { diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index d29aad1e34..5fb933ba5c 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SleeperSystemTest; import sleeper.systemtest.dsl.testutil.InMemoryDslTest; @@ -36,10 +37,11 @@ void setUp(SleeperSystemTest sleeper) { } @Test - void shouldSendOneFileCommit(SleeperSystemTest sleeper) { + void shouldSendOneFileCommit(SleeperSystemTest sleeper) throws Exception { // When sleeper.stateStore().fakeCommits() - .send(factory -> factory.addPartitionFile("root", "file.parquet", 100)); + .send(factory -> factory.addPartitionFile("root", "file.parquet", 100)) + .waitForCommits(PollWithRetries.noRetries()); // Then assertThat(printFiles(sleeper.partitioning().tree(), sleeper.tableFiles().all())) @@ -50,11 +52,12 @@ void shouldSendOneFileCommit(SleeperSystemTest sleeper) { } @Test - void shouldSendManyFileCommits(SleeperSystemTest sleeper) { + void shouldSendManyFileCommits(SleeperSystemTest sleeper) throws Exception { // When sleeper.stateStore().fakeCommits() .sendBatched(factory -> LongStream.rangeClosed(1, 1000) - .mapToObj(i -> factory.addPartitionFile("root", "file-" + i + ".parquet", i))); + .mapToObj(i -> factory.addPartitionFile("root", "file-" + i + ".parquet", i))) + .waitForCommits(PollWithRetries.noRetries()); // Then assertThat(sleeper.tableFiles().references()).hasSize(1000); diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index 536b5bcbde..f858fb4104 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -15,17 +15,26 @@ */ package sleeper.systemtest.dsl.testutil.drivers; +import sleeper.commit.StateStoreCommitRequest; import sleeper.commit.StateStoreCommitter; import sleeper.configuration.properties.table.TablePropertiesProvider; import sleeper.core.statestore.StateStoreException; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; +import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; import java.util.stream.Stream; +import static java.util.stream.Collectors.toUnmodifiableList; + public class InMemoryStateStoreCommitter { private final InMemoryIngestByQueue ingest; @@ -42,6 +51,8 @@ public StateStoreCommitterDriver withContext(SystemTestContext context) { public class Driver implements StateStoreCommitterDriver { private final StateStoreCommitter committer; + private final Queue queue = new LinkedList<>(); + private final List runs = new ArrayList<>(); private Driver(SystemTestContext context) { SystemTestInstanceContext instance = context.instance(); @@ -54,13 +65,32 @@ private Driver(SystemTestContext context) { @Override public void sendCommitMessages(Stream messages) { - messages.forEach(message -> { + messages.forEach(queue::add); + } + + @Override + public List getRunsAfter(Instant startTime) { + runCommitter(); + return runs.stream() + .filter(run -> run.getStartTime().compareTo(startTime) >= 0) + .collect(toUnmodifiableList()); + } + + private void runCommitter() { + Instant startTime = Instant.now(); + List commits = new ArrayList<>(); + for (StateStoreCommitMessage message = queue.poll(); message != null; message = queue.poll()) { try { - committer.applyFromJson(message.getBody()); + StateStoreCommitRequest appliedRequest = committer.applyFromJson(message.getBody()); + commits.add(new StateStoreCommitSummary( + appliedRequest.getTableId(), appliedRequest.getRequest().getClass().getSimpleName(), Instant.now())); } catch (StateStoreException e) { throw new RuntimeException(e); } - }); + } + if (!commits.isEmpty()) { + runs.add(new StateStoreCommitterRun(startTime, Instant.now(), commits)); + } } } diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java index b866089df4..1b60707bed 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -21,11 +21,15 @@ import sleeper.core.partition.PartitionTree; import sleeper.core.partition.PartitionsBuilder; import sleeper.core.statestore.FileReferenceFactory; +import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SleeperSystemTest; import sleeper.systemtest.suite.testutil.SystemTest; +import java.time.Duration; import java.util.stream.IntStream; +import static java.util.stream.Collectors.toUnmodifiableList; +import static org.assertj.core.api.Assertions.assertThat; import static sleeper.systemtest.suite.fixtures.SystemTestInstance.MAIN; import static sleeper.systemtest.suite.fixtures.SystemTestSchema.DEFAULT_SCHEMA; @@ -38,18 +42,23 @@ void setUp(SleeperSystemTest sleeper) { } @Test - void shouldAddManyFiles(SleeperSystemTest sleeper) { + void shouldAddManyFiles(SleeperSystemTest sleeper) throws Exception { // Given PartitionTree partitions = new PartitionsBuilder(DEFAULT_SCHEMA).singlePartition("root").buildTree(); sleeper.partitioning().setPartitions(partitions); // When FileReferenceFactory fileFactory = FileReferenceFactory.from(partitions); - sleeper.stateStore().fakeCommits().sendBatched( - commitFactory -> IntStream.rangeClosed(1, 1000) + sleeper.stateStore().fakeCommits() + .sendBatched(commitFactory -> IntStream.rangeClosed(1, 1000) .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) - .map(commitFactory::addFile)); + .map(commitFactory::addFile)) + .waitForCommits(PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(5), Duration.ofMinutes(2))); // Then + assertThat(sleeper.tableFiles().references()) + .containsExactlyElementsOf(IntStream.rangeClosed(1, 1000) + .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) + .collect(toUnmodifiableList())); } } From 5422fdc8b3b31e1672be700683fa5ff613d278d3 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 11:49:35 +0000 Subject: [PATCH 09/49] Add StateStoreCommitterRunsBuilder --- .../StateStoreCommitterLogEntry.java | 131 ++++++++++++++++++ .../StateStoreCommitterRunsBuilder.java | 101 ++++++++++++++ .../StateStoreCommitterLogTest.java | 55 ++++++++ .../StateStoreCommitterRunsBuilderTest.java | 55 ++++++++ .../statestore/StateStoreCommitSummary.java | 23 +++ .../statestore/StateStoreCommitterRun.java | 28 +++- 6 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java create mode 100644 java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java create mode 100644 java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java create mode 100644 java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java new file mode 100644 index 0000000000..305f320678 --- /dev/null +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -0,0 +1,131 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.drivers.statestore; + +import java.time.Instant; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class StateStoreCommitterLogEntry { + + private static final Pattern MESSAGE_PATTERN = Pattern.compile("Lambda started at (.+)|Lambda finished at ([^ ]+) "); + + private final String message; + private final Object event; + + private StateStoreCommitterLogEntry(String message, Object event) { + this.message = message; + this.event = event; + } + + public static StateStoreCommitterLogEntry from(String message) { + Matcher matcher = MESSAGE_PATTERN.matcher(message); + return new StateStoreCommitterLogEntry(message, readEvent(matcher)); + } + + private static Object readEvent(Matcher matcher) { + if (!matcher.find()) { + return null; + } + String startTime = matcher.group(1); + if (startTime != null) { + return new LambdaStarted(Instant.parse(startTime)); + } + String finishTime = matcher.group(2); + if (finishTime != null) { + return new LambdaFinished(Instant.parse(finishTime)); + } + return null; + } + + public String getMessage() { + return message; + } + + public Object getEvent() { + return event; + } + + public static class LambdaStarted { + private final Instant startTime; + + public LambdaStarted(Instant startTime) { + this.startTime = startTime; + } + + public Instant getStartTime() { + return startTime; + } + + @Override + public int hashCode() { + return Objects.hash(startTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LambdaStarted)) { + return false; + } + LambdaStarted other = (LambdaStarted) obj; + return Objects.equals(startTime, other.startTime); + } + + @Override + public String toString() { + return "LambdaStarted{startTime=" + startTime + "}"; + } + } + + public static class LambdaFinished { + private final Instant finishTime; + + public LambdaFinished(Instant finishTime) { + this.finishTime = finishTime; + } + + public Instant getFinishTime() { + return finishTime; + } + + @Override + public int hashCode() { + return Objects.hash(finishTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LambdaFinished)) { + return false; + } + LambdaFinished other = (LambdaFinished) obj; + return Objects.equals(finishTime, other.finishTime); + } + + @Override + public String toString() { + return "LambdaFinished{finishTime=" + finishTime + "}"; + } + } + +} diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java new file mode 100644 index 0000000000..1c4331bac7 --- /dev/null +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -0,0 +1,101 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.drivers.statestore; + +import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; + +import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaFinished; +import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaStarted; +import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toUnmodifiableList; + +public class StateStoreCommitterRunsBuilder { + + private final Map logStreamByName = new HashMap<>(); + + public void add(List entry) { + LogStream logStream = null; + String message = null; + for (ResultField field : entry) { + switch (field.field()) { + case "@logStream": + logStream = logStreamByName.computeIfAbsent(field.value(), name -> new LogStream()); + case "@message": + message = field.value(); + default: + } + } + Objects.requireNonNull(logStream, "Log stream not found"); + Objects.requireNonNull(message, "Log message not found"); + logStream.add(StateStoreCommitterLogEntry.from(message)); + } + + public List buildRuns() { + return logStreamByName.values().stream() + .flatMap(LogStream::runs) + .map(LambdaRun::build) + .collect(toUnmodifiableList()); + } + + private static class LogStream { + + private final List runs = new ArrayList<>(); + private LambdaRun lastRun; + + void add(StateStoreCommitterLogEntry entry) { + Object event = entry.getEvent(); + if (event instanceof LambdaStarted) { + lastRun = new LambdaRun((LambdaStarted) event); + runs.add(lastRun); + } else if (event instanceof LambdaFinished) { + lastRun.finished((LambdaFinished) event); + } + } + + Stream runs() { + return runs.stream(); + } + } + + private static class LambdaRun { + private final Instant startTime; + private Instant finishTime; + private List commits = new ArrayList<>(); + + LambdaRun(LambdaStarted event) { + this.startTime = event.getStartTime(); + } + + void finished(LambdaFinished event) { + finishTime = event.getFinishTime(); + } + + StateStoreCommitterRun build() { + return new StateStoreCommitterRun(startTime, finishTime, commits); + } + } + +} diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java new file mode 100644 index 0000000000..4036b81350 --- /dev/null +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.drivers.statestore; + +import org.junit.jupiter.api.Test; + +import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaFinished; +import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaStarted; + +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StateStoreCommitterLogTest { + + @Test + void shouldReadLambdaStarted() { + // Given + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + + // When + StateStoreCommitterLogEntry log = StateStoreCommitterLogEntry.from(message); + + // Then + assertThat(log.getEvent()).isEqualTo( + new LambdaStarted(Instant.parse("2024-08-13T12:12:00Z"))); + } + + @Test + void shouldReadLambdaFinished() { + // Given + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"; + + // When + StateStoreCommitterLogEntry log = StateStoreCommitterLogEntry.from(message); + + // Then + assertThat(log.getEvent()).isEqualTo( + new LambdaFinished(Instant.parse("2024-08-13T12:13:00Z"))); + } + +} diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java new file mode 100644 index 0000000000..f5d6a6f6d1 --- /dev/null +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.drivers.statestore; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; + +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StateStoreCommitterRunsBuilderTest { + StateStoreCommitterRunsBuilder builder = new StateStoreCommitterRunsBuilder(); + + @Test + void shouldBuildSingleRunNoCommits() { + add( + "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z", + "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); + } + + void add(String... messages) { + Instant startTime = Instant.now(); + for (int i = 0; i < messages.length; i++) { + add(startTime.plus(Duration.ofMillis(i)), "test-logstream", messages[i]); + } + } + + void add(Instant timestamp, String logStream, String message) { + builder.add(List.of( + ResultField.builder().field("@timestamp").value("" + timestamp.toEpochMilli()).build(), + ResultField.builder().field("@logStream").value(logStream).build(), + ResultField.builder().field("@message").value(message).build())); + } + +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java index 2ba8810192..844559da80 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java @@ -16,6 +16,7 @@ package sleeper.systemtest.dsl.statestore; import java.time.Instant; +import java.util.Objects; public class StateStoreCommitSummary { private final String tableId; @@ -39,4 +40,26 @@ public String getType() { public Instant getFinishTime() { return finishTime; } + + @Override + public int hashCode() { + return Objects.hash(tableId, type, finishTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StateStoreCommitSummary)) { + return false; + } + StateStoreCommitSummary other = (StateStoreCommitSummary) obj; + return Objects.equals(tableId, other.tableId) && Objects.equals(type, other.type) && Objects.equals(finishTime, other.finishTime); + } + + @Override + public String toString() { + return "StateStoreCommitSummary{tableId=" + tableId + ", type=" + type + ", finishTime=" + finishTime + "}"; + } } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java index 27010627d1..b35a291b6f 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java @@ -17,10 +17,7 @@ import java.time.Instant; import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.summingInt; +import java.util.Objects; public class StateStoreCommitterRun { @@ -46,8 +43,25 @@ public List getCommits() { return commits; } - public Map countCommitsByTableId() { - return commits.stream() - .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); + @Override + public int hashCode() { + return Objects.hash(startTime, finishTime, commits); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StateStoreCommitterRun)) { + return false; + } + StateStoreCommitterRun other = (StateStoreCommitterRun) obj; + return Objects.equals(startTime, other.startTime) && Objects.equals(finishTime, other.finishTime) && Objects.equals(commits, other.commits); + } + + @Override + public String toString() { + return "StateStoreCommitterRun{startTime=" + startTime + ", finishTime=" + finishTime + ", commits=" + commits + "}"; } } From 72b487e35bea894bad17cc1f05a6a797f26bfa48 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:04:48 +0000 Subject: [PATCH 10/49] Read state store commit applied event --- .../sleeper/commit/StateStoreCommitter.java | 4 +- .../StateStoreCommitterLogEntry.java | 31 +++++-------- .../StateStoreCommitterRunsBuilder.java | 15 ++++--- ...a => StateStoreCommitterLogEntryTest.java} | 26 ++++++----- .../StateStoreCommitterRunsBuilderTest.java | 43 ++++++++++++++----- 5 files changed, 70 insertions(+), 49 deletions(-) rename java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/{StateStoreCommitterLogTest.java => StateStoreCommitterLogEntryTest.java} (66%) diff --git a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java index ffcde2a857..ca252acec6 100644 --- a/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java +++ b/java/statestore-commit/src/main/java/sleeper/commit/StateStoreCommitter.java @@ -93,8 +93,8 @@ public StateStoreCommitRequest applyFromJson(String json) throws StateStoreExcep */ public void apply(StateStoreCommitRequest request) throws StateStoreException { request.apply(this); - LOGGER.info("Applied request to table ID {} with type {}", - request.getTableId(), request.getRequest().getClass().getSimpleName()); + LOGGER.info("Applied request to table ID {} with type {} at time {}", + request.getTableId(), request.getRequest().getClass().getSimpleName(), Instant.now()); } void commitCompaction(CompactionJobCommitRequest request) throws StateStoreException { diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java index 305f320678..1731effcb6 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -15,6 +15,8 @@ */ package sleeper.systemtest.drivers.statestore; +import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; + import java.time.Instant; import java.util.Objects; import java.util.regex.Matcher; @@ -22,22 +24,13 @@ public class StateStoreCommitterLogEntry { - private static final Pattern MESSAGE_PATTERN = Pattern.compile("Lambda started at (.+)|Lambda finished at ([^ ]+) "); - - private final String message; - private final Object event; + private static final Pattern MESSAGE_PATTERN = Pattern.compile("Lambda started at (.+)|Lambda finished at ([^ ]+) |Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); - private StateStoreCommitterLogEntry(String message, Object event) { - this.message = message; - this.event = event; + private StateStoreCommitterLogEntry() { } - public static StateStoreCommitterLogEntry from(String message) { + public static Object readEvent(String message) { Matcher matcher = MESSAGE_PATTERN.matcher(message); - return new StateStoreCommitterLogEntry(message, readEvent(matcher)); - } - - private static Object readEvent(Matcher matcher) { if (!matcher.find()) { return null; } @@ -49,17 +42,15 @@ private static Object readEvent(Matcher matcher) { if (finishTime != null) { return new LambdaFinished(Instant.parse(finishTime)); } + String tableId = matcher.group(3); + if (tableId != null) { + String type = matcher.group(4); + String commitTime = matcher.group(5); + return new StateStoreCommitSummary(tableId, type, Instant.parse(commitTime)); + } return null; } - public String getMessage() { - return message; - } - - public Object getEvent() { - return event; - } - public static class LambdaStarted { private final Instant startTime; diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java index 1c4331bac7..fa2aa41dcc 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -24,7 +24,7 @@ import java.time.Instant; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -34,7 +34,7 @@ public class StateStoreCommitterRunsBuilder { - private final Map logStreamByName = new HashMap<>(); + private final Map logStreamByName = new LinkedHashMap<>(); public void add(List entry) { LogStream logStream = null; @@ -50,7 +50,7 @@ public void add(List entry) { } Objects.requireNonNull(logStream, "Log stream not found"); Objects.requireNonNull(message, "Log message not found"); - logStream.add(StateStoreCommitterLogEntry.from(message)); + logStream.add(StateStoreCommitterLogEntry.readEvent(message)); } public List buildRuns() { @@ -65,13 +65,14 @@ private static class LogStream { private final List runs = new ArrayList<>(); private LambdaRun lastRun; - void add(StateStoreCommitterLogEntry entry) { - Object event = entry.getEvent(); + void add(Object event) { if (event instanceof LambdaStarted) { lastRun = new LambdaRun((LambdaStarted) event); runs.add(lastRun); } else if (event instanceof LambdaFinished) { lastRun.finished((LambdaFinished) event); + } else if (event instanceof StateStoreCommitSummary) { + lastRun.committed((StateStoreCommitSummary) event); } } @@ -93,6 +94,10 @@ void finished(LambdaFinished event) { finishTime = event.getFinishTime(); } + void committed(StateStoreCommitSummary commit) { + commits.add(commit); + } + StateStoreCommitterRun build() { return new StateStoreCommitterRun(startTime, finishTime, commits); } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java similarity index 66% rename from java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java rename to java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java index 4036b81350..b4922b723a 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java @@ -19,23 +19,21 @@ import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaFinished; import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaStarted; +import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import java.time.Instant; import static org.assertj.core.api.Assertions.assertThat; -public class StateStoreCommitterLogTest { +public class StateStoreCommitterLogEntryTest { @Test void shouldReadLambdaStarted() { // Given String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; - // When - StateStoreCommitterLogEntry log = StateStoreCommitterLogEntry.from(message); - - // Then - assertThat(log.getEvent()).isEqualTo( + // When / Then + assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( new LambdaStarted(Instant.parse("2024-08-13T12:12:00Z"))); } @@ -44,12 +42,18 @@ void shouldReadLambdaFinished() { // Given String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"; - // When - StateStoreCommitterLogEntry log = StateStoreCommitterLogEntry.from(message); - - // Then - assertThat(log.getEvent()).isEqualTo( + // When / Then + assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( new LambdaFinished(Instant.parse("2024-08-13T12:13:00Z"))); } + @Test + void shouldReadCommitApplied() { + // Given + String message = "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"; + + // When / Then + assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( + new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))); + } } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index f5d6a6f6d1..9b2a9aae78 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -18,9 +18,9 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; +import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; -import java.time.Duration; import java.time.Instant; import java.util.List; @@ -31,23 +31,44 @@ public class StateStoreCommitterRunsBuilderTest { @Test void shouldBuildSingleRunNoCommits() { - add( - "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z", - "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); + // Given + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); + + // When / Then assertThat(builder.buildRuns()).containsExactly( new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); } - void add(String... messages) { - Instant startTime = Instant.now(); - for (int i = 0; i < messages.length; i++) { - add(startTime.plus(Duration.ofMillis(i)), "test-logstream", messages[i]); - } + @Test + void shouldBuildOverlappingRunsOnTwoLogStreams() { + // Given + add("stream-1", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); + add("stream-2", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:30Z"); + add("stream-1", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); + add("stream-2", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:30Z (ran for 1 minute)"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of()), + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:30Z"), Instant.parse("2024-08-13T12:13:30Z"), List.of())); + } + + @Test + void shouldBuildSingleRunWithOneCommit() { + // Given + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); + add("test-logstream", "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"); + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), + List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } - void add(Instant timestamp, String logStream, String message) { + void add(String logStream, String message) { builder.add(List.of( - ResultField.builder().field("@timestamp").value("" + timestamp.toEpochMilli()).build(), ResultField.builder().field("@logStream").value(logStream).build(), ResultField.builder().field("@message").value(message).build())); } From 6206a31d4e82dd754e548c07212a97b4469a5fff Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:06:55 +0000 Subject: [PATCH 11/49] Test reading unrecognised log --- .../statestore/StateStoreCommitterLogEntryTest.java | 9 +++++++++ .../statestore/StateStoreCommitterRunsBuilderTest.java | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java index b4922b723a..e11fc03548 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java @@ -56,4 +56,13 @@ void shouldReadCommitApplied() { assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))); } + + @Test + void shouldReadUnrecognisedLog() { + // Given + String message = "some other log"; + + // When / Then + assertThat(StateStoreCommitterLogEntry.readEvent(message)).isNull(); + } } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index 9b2a9aae78..51c87ec487 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -67,6 +67,15 @@ void shouldBuildSingleRunWithOneCommit() { List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } + @Test + void shouldIgnoreUnrecognisedLog() { + // Given + add("test-logstream", "Some unknown message"); + + // When / Then + assertThat(builder.buildRuns()).isEmpty(); + } + void add(String logStream, String message) { builder.add(List.of( ResultField.builder().field("@logStream").value(logStream).build(), From 8bf28046a92ee85845ee651dcd7c59c59dd2e311 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:14:44 +0000 Subject: [PATCH 12/49] Read logs in AwsStateStoreCommitterDriver --- .../cdk/stack/StateStoreCommitterStack.java | 6 +- .../instance/CdkDefinedInstanceProperty.java | 4 ++ .../AwsStateStoreCommitterDriver.java | 56 ++++++++++++++++++- .../drivers/util/AwsSystemTestDrivers.java | 2 +- .../drivers/util/SystemTestClients.java | 15 +++++ 5 files changed, 78 insertions(+), 5 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java index f40651856c..d94bcdefd3 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java @@ -21,6 +21,7 @@ import software.amazon.awscdk.services.iam.IGrantable; import software.amazon.awscdk.services.lambda.IFunction; import software.amazon.awscdk.services.lambda.eventsources.SqsEventSource; +import software.amazon.awscdk.services.logs.LogGroup; import software.amazon.awscdk.services.s3.Bucket; import software.amazon.awscdk.services.s3.IBucket; import software.amazon.awscdk.services.sns.Topic; @@ -41,6 +42,7 @@ import static sleeper.cdk.Utils.createLambdaLogGroup; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_DLQ_ARN; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_DLQ_URL; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_LOG_GROUP; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_QUEUE_ARN; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_QUEUE_URL; import static sleeper.configuration.properties.instance.CommonProperty.STATESTORE_COMMITTER_BATCH_SIZE; @@ -116,6 +118,8 @@ private void lambdaToCommitStateStoreUpdates( String functionName = String.join("-", "sleeper", Utils.cleanInstanceId(instanceProperties), "statestore-committer"); + LogGroup logGroup = createLambdaLogGroup(this, "StateStoreCommitterLogGroup", functionName, instanceProperties); + instanceProperties.set(STATESTORE_COMMITTER_LOG_GROUP, logGroup.getLogGroupName()); IFunction handlerFunction = committerJar.buildFunction(this, "StateStoreCommitter", builder -> builder .functionName(functionName) @@ -125,7 +129,7 @@ private void lambdaToCommitStateStoreUpdates( .timeout(Duration.seconds(instanceProperties.getInt(STATESTORE_COMMITTER_LAMBDA_TIMEOUT_IN_SECONDS))) .handler("sleeper.statestore.committer.lambda.StateStoreCommitterLambda::handleRequest") .environment(environmentVariables) - .logGroup(createLambdaLogGroup(this, "StateStoreCommitterLogGroup", functionName, instanceProperties))); + .logGroup(logGroup)); handlerFunction.addEventSource(SqsEventSource.Builder.create(commitQueue) .batchSize(instanceProperties.getInt(STATESTORE_COMMITTER_BATCH_SIZE)) diff --git a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java index da2aaf74ef..24947ccfda 100644 --- a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java +++ b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java @@ -179,6 +179,10 @@ public interface CdkDefinedInstanceProperty extends InstanceProperty { .description("The ARN of the dead letter queue for statestore commit requests.") .propertyGroup(InstancePropertyGroup.COMMON) .build(); + CdkDefinedInstanceProperty STATESTORE_COMMITTER_LOG_GROUP = Index.propertyBuilder("sleeper.statestore.committer.log.group") + .description("The nane of the log group for the state store committer.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); // Table metrics CdkDefinedInstanceProperty TABLE_METRICS_LAMBDA_FUNCTION = Index.propertyBuilder("sleeper.table.metrics.lambda.function") diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java index 759ac83be7..2c8f621270 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java @@ -18,29 +18,41 @@ import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.model.SendMessageBatchRequest; import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; +import software.amazon.awssdk.services.cloudwatchlogs.model.GetQueryResultsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.QueryStatus; +import sleeper.core.util.PollWithRetries; import sleeper.core.util.SplitIntoBatches; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; +import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.stream.Stream; import static java.util.stream.Collectors.toUnmodifiableList; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_LOG_GROUP; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_QUEUE_URL; public class AwsStateStoreCommitterDriver implements StateStoreCommitterDriver { + public static final Logger LOGGER = LoggerFactory.getLogger(AwsStateStoreCommitterDriver.class); private final SystemTestInstanceContext instance; private final AmazonSQS sqs; + private final CloudWatchLogsClient cloudWatch; - public AwsStateStoreCommitterDriver(SystemTestInstanceContext instance, AmazonSQS sqs) { + public AwsStateStoreCommitterDriver(SystemTestInstanceContext instance, AmazonSQS sqs, CloudWatchLogsClient cloudWatch) { this.instance = instance; this.sqs = sqs; + this.cloudWatch = cloudWatch; } @Override @@ -50,8 +62,21 @@ public void sendCommitMessages(Stream messages) { @Override public List getRunsAfter(Instant startTime) { - // TODO Auto-generated method stub - return null; + String logGroupName = instance.getInstanceProperties().get(STATESTORE_COMMITTER_LOG_GROUP); + LOGGER.info("Submitting logs query for log group {} starting at time {}", logGroupName, startTime); + String queryId = cloudWatch.startQuery(builder -> builder + .logGroupName(logGroupName) + .startTime(startTime.getEpochSecond()) + .endTime(Instant.now().plus(Duration.ofMinutes(1)).getEpochSecond()) + .limit(10000) + .queryString("fields @timestamp, @message, @logStream " + + "| filter @message like /Lambda (started|finished) at|Applied request to table/ " + + "| sort @timestamp asc")) + .queryId(); + GetQueryResultsResponse response = waitForQuery(queryId); + StateStoreCommitterRunsBuilder builder = new StateStoreCommitterRunsBuilder(); + response.results().forEach(builder::add); + return builder.buildRuns(); } private void sendMessageBatch(List batch) { @@ -66,4 +91,29 @@ private void sendMessageBatch(List batch) { .collect(toUnmodifiableList()))); } + private GetQueryResultsResponse waitForQuery(String queryId) { + try { + return PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(1), Duration.ofMinutes(1)) + .queryUntil("query is completed", + () -> cloudWatch.getQueryResults(builder -> builder.queryId(queryId)), + results -> isQueryCompleted(results)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static boolean isQueryCompleted(GetQueryResultsResponse response) { + LOGGER.info("Logs query response status {}, statistics: {}", + response.statusAsString(), response.statistics()); + QueryStatus status = response.status(); + if (status == QueryStatus.COMPLETE) { + return true; + } else if (Set.of(QueryStatus.SCHEDULED, QueryStatus.RUNNING).contains(status)) { + return false; + } else { + throw new RuntimeException("Logs query failed with status " + response.statusAsString()); + } + } + } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java index 99af68d260..bb24ff4c8c 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java @@ -112,7 +112,7 @@ public SleeperTablesDriver tables(SystemTestParameters parameters) { @Override public StateStoreCommitterDriver stateStoreCommitter(SystemTestContext context) { - return new AwsStateStoreCommitterDriver(context.instance(), clients.getSqs()); + return new AwsStateStoreCommitterDriver(context.instance(), clients.getSqs(), clients.getCloudWatchLogs()); } @Override diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/SystemTestClients.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/SystemTestClients.java index 81bfcdc92f..777930a4ed 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/SystemTestClients.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/SystemTestClients.java @@ -40,6 +40,7 @@ import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.cloudformation.CloudFormationClient; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.emrserverless.EmrServerlessClient; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.lambda.LambdaClientBuilder; @@ -78,6 +79,7 @@ public class SystemTestClients { private final AmazonAutoScaling autoScaling; private final AmazonECR ecr; private final CloudWatchClient cloudWatch; + private final CloudWatchLogsClient cloudWatchLogs; private final AmazonCloudWatchEvents cloudWatchEvents; private final Supplier> getAuthEnvVars; private final UnaryOperator configureHadoop; @@ -100,6 +102,7 @@ private SystemTestClients(Builder builder) { autoScaling = builder.autoScaling; ecr = builder.ecr; cloudWatch = builder.cloudWatch; + cloudWatchLogs = builder.cloudWatchLogs; cloudWatchEvents = builder.cloudWatchEvents; getAuthEnvVars = builder.getAuthEnvVars; configureHadoop = builder.configureHadoop; @@ -128,6 +131,7 @@ public static SystemTestClients fromDefaults() { .autoScaling(AmazonAutoScalingClientBuilder.defaultClient()) .ecr(AmazonECRClientBuilder.defaultClient()) .cloudWatch(CloudWatchClient.create()) + .cloudWatchLogs(CloudWatchLogsClient.create()) .cloudWatchEvents(AmazonCloudWatchEventsClientBuilder.defaultClient()) .build(); } @@ -156,6 +160,7 @@ public SystemTestClients assumeRole(AssumeSleeperRole assumeRole) { .autoScaling(v1.buildClient(AmazonAutoScalingClientBuilder.standard())) .ecr(v1.buildClient(AmazonECRClientBuilder.standard())) .cloudWatch(v2.buildClient(CloudWatchClient.builder())) + .cloudWatchLogs(v2.buildClient(CloudWatchLogsClient.builder())) .cloudWatchEvents(v1.buildClient(AmazonCloudWatchEventsClientBuilder.standard())) .getAuthEnvVars(v1::authEnvVars) .configureHadoop(hadoop::setS3ACredentials) @@ -222,6 +227,10 @@ public CloudWatchClient getCloudWatch() { return cloudWatch; } + public CloudWatchLogsClient getCloudWatchLogs() { + return cloudWatchLogs; + } + public AmazonCloudWatchEvents getCloudWatchEvents() { return cloudWatchEvents; } @@ -264,6 +273,7 @@ public static class Builder { private AmazonAutoScaling autoScaling; private AmazonECR ecr; private CloudWatchClient cloudWatch; + private CloudWatchLogsClient cloudWatchLogs; private AmazonCloudWatchEvents cloudWatchEvents; private Supplier> getAuthEnvVars = Map::of; private UnaryOperator configureHadoop = conf -> conf; @@ -352,6 +362,11 @@ public Builder cloudWatch(CloudWatchClient cloudWatch) { return this; } + public Builder cloudWatchLogs(CloudWatchLogsClient cloudWatchLogs) { + this.cloudWatchLogs = cloudWatchLogs; + return this; + } + public Builder cloudWatchEvents(AmazonCloudWatchEvents cloudWatchEvents) { this.cloudWatchEvents = cloudWatchEvents; return this; From adb98ec5358b03101915104e77cccec78b086121 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:18:37 +0000 Subject: [PATCH 13/49] Fix reading log with fields in reverse order --- .../StateStoreCommitterRunsBuilder.java | 2 ++ .../StateStoreCommitterRunsBuilderTest.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java index fa2aa41dcc..1388d9339f 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -43,8 +43,10 @@ public void add(List entry) { switch (field.field()) { case "@logStream": logStream = logStreamByName.computeIfAbsent(field.value(), name -> new LogStream()); + break; case "@message": message = field.value(); + break; default: } } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index 51c87ec487..0669c7f657 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -76,10 +76,26 @@ void shouldIgnoreUnrecognisedLog() { assertThat(builder.buildRuns()).isEmpty(); } + @Test + void shouldReadLogWithFieldsInReverseOrder() { + // Given + addReversingFields("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); + addReversingFields("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); + } + void add(String logStream, String message) { builder.add(List.of( ResultField.builder().field("@logStream").value(logStream).build(), ResultField.builder().field("@message").value(message).build())); } + void addReversingFields(String logStream, String message) { + builder.add(List.of( + ResultField.builder().field("@message").value(message).build(), + ResultField.builder().field("@logStream").value(logStream).build())); + } } From 6a732943405a18d0fa3708e8aee4b8eb40d7d702 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:21:44 +0000 Subject: [PATCH 14/49] Test ignoring extra log fields --- .../StateStoreCommitterRunsBuilder.java | 1 + .../StateStoreCommitterRunsBuilderTest.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java index 1388d9339f..e6d9684be0 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -48,6 +48,7 @@ public void add(List entry) { message = field.value(); break; default: + break; } } Objects.requireNonNull(logStream, "Log stream not found"); diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index 0669c7f657..6a1bca6957 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -87,6 +87,23 @@ void shouldReadLogWithFieldsInReverseOrder() { new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); } + @Test + void shouldIgnoreExtraFieldsInLogMessage() { + // Given + builder.add(List.of( + ResultField.builder().field("@logStream").value("test-logstream").build(), + ResultField.builder().field("@someField").value("Some ignored field").build(), + ResultField.builder().field("@message").value("[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z").build())); + builder.add(List.of( + ResultField.builder().field("@logStream").value("test-logstream").build(), + ResultField.builder().field("@message").value("[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)").build(), + ResultField.builder().field("@otherField").value("Other ignored field").build())); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); + } + void add(String logStream, String message) { builder.add(List.of( ResultField.builder().field("@logStream").value(logStream).build(), From d2b2b180b623c5ca1552ad8ea36831e732d56b21 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 07:51:27 +0000 Subject: [PATCH 15/49] Add comments to StateStoreCommitterLogEntry --- .../drivers/statestore/StateStoreCommitterLogEntry.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java index 1731effcb6..a63e7a57a4 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -24,7 +24,10 @@ public class StateStoreCommitterLogEntry { - private static final Pattern MESSAGE_PATTERN = Pattern.compile("Lambda started at (.+)|Lambda finished at ([^ ]+) |Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); + private static final Pattern MESSAGE_PATTERN = Pattern.compile("" + + "Lambda started at (.+)|" + // Lambda started message type has capture group 1 + "Lambda finished at ([^ ]+) |" + // Lambda finished message type has capture group 2 + "Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); // Capture groups 3, 4 and 5 private StateStoreCommitterLogEntry() { } @@ -34,6 +37,9 @@ public static Object readEvent(String message) { if (!matcher.find()) { return null; } + // The pattern can only match one type of log message at a time. + // Each capture group will be null unless its message type was matched. + // We determine which type of message was found based on which capture group is set. String startTime = matcher.group(1); if (startTime != null) { return new LambdaStarted(Instant.parse(startTime)); From a263ba236f740afa1be748f6bb0314c556e6aea6 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 07:53:52 +0000 Subject: [PATCH 16/49] Use named log groups in StateStoreCommitterLogEntry --- .../statestore/StateStoreCommitterLogEntry.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java index a63e7a57a4..81ac136531 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -25,9 +25,9 @@ public class StateStoreCommitterLogEntry { private static final Pattern MESSAGE_PATTERN = Pattern.compile("" + - "Lambda started at (.+)|" + // Lambda started message type has capture group 1 - "Lambda finished at ([^ ]+) |" + // Lambda finished message type has capture group 2 - "Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); // Capture groups 3, 4 and 5 + "Lambda started at (?.+)|" + // Lambda started message type + "Lambda finished at (?[^ ]+) |" + // Lambda finished message type + "Applied request to table ID (?[^ ]+) with type (?[^ ]+) at time (?[^ ]+)"); // Commit applied message type private StateStoreCommitterLogEntry() { } @@ -40,18 +40,18 @@ public static Object readEvent(String message) { // The pattern can only match one type of log message at a time. // Each capture group will be null unless its message type was matched. // We determine which type of message was found based on which capture group is set. - String startTime = matcher.group(1); + String startTime = matcher.group("startTime"); if (startTime != null) { return new LambdaStarted(Instant.parse(startTime)); } - String finishTime = matcher.group(2); + String finishTime = matcher.group("finishTime"); if (finishTime != null) { return new LambdaFinished(Instant.parse(finishTime)); } - String tableId = matcher.group(3); + String tableId = matcher.group("tableId"); if (tableId != null) { - String type = matcher.group(4); - String commitTime = matcher.group(5); + String type = matcher.group("type"); + String commitTime = matcher.group("commitTime"); return new StateStoreCommitSummary(tableId, type, Instant.parse(commitTime)); } return null; From 4b8eb463e04e0d8711651aceb37986c866659555 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 07:58:04 +0000 Subject: [PATCH 17/49] Use constants for capture groups --- .../StateStoreCommitterLogEntry.java | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java index 81ac136531..b2a4810808 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -25,9 +25,20 @@ public class StateStoreCommitterLogEntry { private static final Pattern MESSAGE_PATTERN = Pattern.compile("" + - "Lambda started at (?.+)|" + // Lambda started message type - "Lambda finished at (?[^ ]+) |" + // Lambda finished message type - "Applied request to table ID (?[^ ]+) with type (?[^ ]+) at time (?[^ ]+)"); // Commit applied message type + "Lambda started at (.+)|" + // Lambda started message type + "Lambda finished at ([^ ]+) |" + // Lambda finished message type + "Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); // Commit applied message type + + private static class CaptureGroups { + private static final int START_TIME = 1; + private static final int FINISH_TIME = 2; + private static final int TABLE_ID = 3; + private static final int TYPE = 4; + private static final int COMMIT_TIME = 5; + + private CaptureGroups() { + } + } private StateStoreCommitterLogEntry() { } @@ -40,18 +51,18 @@ public static Object readEvent(String message) { // The pattern can only match one type of log message at a time. // Each capture group will be null unless its message type was matched. // We determine which type of message was found based on which capture group is set. - String startTime = matcher.group("startTime"); + String startTime = matcher.group(CaptureGroups.START_TIME); if (startTime != null) { return new LambdaStarted(Instant.parse(startTime)); } - String finishTime = matcher.group("finishTime"); + String finishTime = matcher.group(CaptureGroups.FINISH_TIME); if (finishTime != null) { return new LambdaFinished(Instant.parse(finishTime)); } - String tableId = matcher.group("tableId"); + String tableId = matcher.group(CaptureGroups.TABLE_ID); if (tableId != null) { - String type = matcher.group("type"); - String commitTime = matcher.group("commitTime"); + String type = matcher.group(CaptureGroups.TYPE); + String commitTime = matcher.group(CaptureGroups.COMMIT_TIME); return new StateStoreCommitSummary(tableId, type, Instant.parse(commitTime)); } return null; From 35f7b5259dd5f1eb633385eb0d7cc351215cc3af Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:00:04 +0000 Subject: [PATCH 18/49] Replace "capture group" with "capturing group" --- .../StateStoreCommitterLogEntry.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java index b2a4810808..4e9101b665 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -29,14 +29,14 @@ public class StateStoreCommitterLogEntry { "Lambda finished at ([^ ]+) |" + // Lambda finished message type "Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); // Commit applied message type - private static class CaptureGroups { + private static class CapturingGroups { private static final int START_TIME = 1; private static final int FINISH_TIME = 2; private static final int TABLE_ID = 3; private static final int TYPE = 4; private static final int COMMIT_TIME = 5; - private CaptureGroups() { + private CapturingGroups() { } } @@ -49,20 +49,20 @@ public static Object readEvent(String message) { return null; } // The pattern can only match one type of log message at a time. - // Each capture group will be null unless its message type was matched. - // We determine which type of message was found based on which capture group is set. - String startTime = matcher.group(CaptureGroups.START_TIME); + // Each capturing group will be null unless its message type was matched. + // We determine which type of message was found based on which capturing group is set. + String startTime = matcher.group(CapturingGroups.START_TIME); if (startTime != null) { return new LambdaStarted(Instant.parse(startTime)); } - String finishTime = matcher.group(CaptureGroups.FINISH_TIME); + String finishTime = matcher.group(CapturingGroups.FINISH_TIME); if (finishTime != null) { return new LambdaFinished(Instant.parse(finishTime)); } - String tableId = matcher.group(CaptureGroups.TABLE_ID); + String tableId = matcher.group(CapturingGroups.TABLE_ID); if (tableId != null) { - String type = matcher.group(CaptureGroups.TYPE); - String commitTime = matcher.group(CaptureGroups.COMMIT_TIME); + String type = matcher.group(CapturingGroups.TYPE); + String commitTime = matcher.group(CapturingGroups.COMMIT_TIME); return new StateStoreCommitSummary(tableId, type, Instant.parse(commitTime)); } return null; From 44fe0a608becad6be787ef69864d7e78c7997d7d Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:04:08 +0000 Subject: [PATCH 19/49] Load from S3 in AwsStateStoreCommitterDriverIT --- .../statestore/AwsStateStoreCommitterDriverIT.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java index 804cc7cae6..b6c273c565 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriverIT.java @@ -15,6 +15,7 @@ */ package sleeper.systemtest.drivers.statestore; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.model.Message; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; @@ -23,7 +24,6 @@ import sleeper.commit.StateStoreCommitRequest; import sleeper.commit.StateStoreCommitRequestDeserialiser; -import sleeper.commit.StateStoreCommitter.LoadS3ObjectFromDataBucket; import sleeper.core.partition.PartitionTree; import sleeper.core.partition.PartitionsBuilder; import sleeper.core.statestore.FileReference; @@ -42,6 +42,7 @@ import static java.util.stream.Collectors.toUnmodifiableList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.DATA_BUCKET; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_QUEUE_URL; import static sleeper.configuration.properties.table.TableProperty.TABLE_ID; import static sleeper.systemtest.drivers.testutil.LocalStackTestInstance.DEFAULT_SCHEMA; @@ -51,12 +52,14 @@ public class AwsStateStoreCommitterDriverIT { private AmazonSQS sqs; + private AmazonS3 s3; private SystemTestInstanceContext instance; @BeforeEach void setUp(SleeperSystemTest sleeper, SystemTestContext context, LocalStackSystemTestDrivers drivers) { sleeper.connectToInstance(MAIN); sqs = drivers.clients().getSqs(); + s3 = drivers.clients().getS3(); instance = context.instance(); } @@ -130,10 +133,8 @@ private String getMessageGroupId(Message message) { } private StateStoreCommitRequest readCommitRequest(Message message) { - LoadS3ObjectFromDataBucket noLoadFromS3 = key -> { - throw new UnsupportedOperationException("Did not expect message held in S3"); - }; - return new StateStoreCommitRequestDeserialiser(instance.getTablePropertiesProvider(), noLoadFromS3) + return new StateStoreCommitRequestDeserialiser(instance.getTablePropertiesProvider(), + key -> s3.getObjectAsString(instance.getInstanceProperties().get(DATA_BUCKET), key)) .fromJson(message.getBody()); } From eda82128119a834067d9b1cea499c27b204c3edc Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:09:59 +0000 Subject: [PATCH 20/49] Test building unfinished runs --- .../StateStoreCommitterRunsBuilderTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index 6a1bca6957..e6bfb1944f 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -67,6 +67,28 @@ void shouldBuildSingleRunWithOneCommit() { List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } + @Test + void shouldBuildUnfinishedRun() { + // Given + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), null, List.of())); + } + + @Test + void shouldBuildUnfinishedRunWithOneCommit() { + // Given + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); + add("test-logstream", "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), null, + List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); + } + @Test void shouldIgnoreUnrecognisedLog() { // Given From 769bdd469c62b93f307410003d66670be4a0b8b8 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:22:51 +0000 Subject: [PATCH 21/49] Fix wait for commit --- .../statestore/StateStoreCommitterRun.java | 11 ++++++++ .../SystemTestStateStoreFakeCommits.java | 2 +- .../SystemTestStateStoreFakeCommitsTest.java | 25 ++++++++++++++++++- .../testutil/InMemorySystemTestDrivers.java | 10 +++++--- .../drivers/InMemoryStateStoreCommitter.java | 17 ++++++++++--- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java index b35a291b6f..79c17c55a3 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java @@ -16,6 +16,7 @@ package sleeper.systemtest.dsl.statestore; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.Objects; @@ -39,6 +40,16 @@ public Instant getFinishTime() { return finishTime; } + public Instant getLastTime() { + if (finishTime != null) { + return finishTime; + } + return commits.stream() + .map(StateStoreCommitSummary::getFinishTime) + .max(Comparator.naturalOrder()) + .orElse(startTime); + } + public List getCommits() { return commits; } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index d912443c0c..70f1ae5e2b 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -89,7 +89,7 @@ private void decrementWaitForNumCommits(List runs) { private void updateFindCommitsFromTime(List runs) { runs.stream() - .map(StateStoreCommitterRun::getFinishTime) + .map(StateStoreCommitterRun::getLastTime) .max(Comparator.naturalOrder()) .ifPresent(lastTime -> findCommitsFromTime = lastTime); } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index 5fb933ba5c..a47cbb054f 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -21,19 +21,28 @@ import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SleeperSystemTest; import sleeper.systemtest.dsl.testutil.InMemoryDslTest; +import sleeper.systemtest.dsl.testutil.InMemorySystemTestDrivers; +import sleeper.systemtest.dsl.testutil.drivers.InMemoryStateStoreCommitter; +import java.time.Instant; +import java.util.List; import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static sleeper.configuration.properties.table.TableProperty.TABLE_ID; import static sleeper.core.testutils.printers.FileReferencePrinter.printFiles; import static sleeper.systemtest.dsl.testutil.InMemoryTestInstance.MAIN; @InMemoryDslTest public class SystemTestStateStoreFakeCommitsTest { + private InMemoryStateStoreCommitter committer; + @BeforeEach - void setUp(SleeperSystemTest sleeper) { + void setUp(SleeperSystemTest sleeper, InMemorySystemTestDrivers drivers) { sleeper.connectToInstance(MAIN); + committer = drivers.stateStoreCommitter(); } @Test @@ -63,4 +72,18 @@ void shouldSendManyFileCommits(SleeperSystemTest sleeper) throws Exception { assertThat(sleeper.tableFiles().references()).hasSize(1000); } + @Test + void shouldWaitForCommitWhenCommitWasMadeButRunIsUnfinished(SleeperSystemTest sleeper) throws Exception { + // Given + committer.setRunCommitterOnSend(false); + SystemTestStateStoreFakeCommits commitsDsl = sleeper.stateStore().fakeCommits(); + commitsDsl.send(factory -> factory.addPartitionFile("root", "file.parquet", 100)); + committer.addRun(new StateStoreCommitterRun(Instant.now(), null, + List.of(new StateStoreCommitSummary(sleeper.tableProperties().get(TABLE_ID), "test-file-added", Instant.now())))); + + // When / Then + assertThatCode(() -> commitsDsl.waitForCommits(PollWithRetries.noRetries())) + .doesNotThrowAnyException(); + } + } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java index 3e3a052205..d383a945b0 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java @@ -209,6 +209,11 @@ public PartitionReportDriver partitionReports(SystemTestContext context) { return reports.partitions(context.instance()); } + @Override + public SnapshotsDriver snapshots() { + return new InMemorySnapshotsDriver(); + } + public InMemoryReports reports() { return reports; } @@ -217,9 +222,8 @@ public InMemoryDataStore data() { return data; } - @Override - public SnapshotsDriver snapshots() { - return new InMemorySnapshotsDriver(); + public InMemoryStateStoreCommitter stateStoreCommitter() { + return stateStoreCommitter; } } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index f858fb4104..66dc3c55eb 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -39,6 +39,9 @@ public class InMemoryStateStoreCommitter { private final InMemoryIngestByQueue ingest; private final InMemoryCompaction compaction; + private final Queue queue = new LinkedList<>(); + private final List runs = new ArrayList<>(); + private boolean runCommitterOnSend = true; public InMemoryStateStoreCommitter(InMemoryIngestByQueue ingest, InMemoryCompaction compaction) { this.ingest = ingest; @@ -49,10 +52,16 @@ public StateStoreCommitterDriver withContext(SystemTestContext context) { return new Driver(context); } + public void setRunCommitterOnSend(boolean runCommitterOnSend) { + this.runCommitterOnSend = runCommitterOnSend; + } + + public void addRun(StateStoreCommitterRun run) { + runs.add(run); + } + public class Driver implements StateStoreCommitterDriver { private final StateStoreCommitter committer; - private final Queue queue = new LinkedList<>(); - private final List runs = new ArrayList<>(); private Driver(SystemTestContext context) { SystemTestInstanceContext instance = context.instance(); @@ -66,11 +75,13 @@ private Driver(SystemTestContext context) { @Override public void sendCommitMessages(Stream messages) { messages.forEach(queue::add); + if (runCommitterOnSend) { + runCommitter(); + } } @Override public List getRunsAfter(Instant startTime) { - runCommitter(); return runs.stream() .filter(run -> run.getStartTime().compareTo(startTime) >= 0) .collect(toUnmodifiableList()); From 24a4e6762ac5f04f95ad3c843543806025439286 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:47:33 +0000 Subject: [PATCH 22/49] Fix SystemTestStateStoreFakeCommitsTest --- .../SystemTestStateStoreFakeCommitsTest.java | 4 ++-- .../drivers/InMemoryStateStoreCommitter.java | 24 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index a47cbb054f..44d501ab75 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -75,10 +75,10 @@ void shouldSendManyFileCommits(SleeperSystemTest sleeper) throws Exception { @Test void shouldWaitForCommitWhenCommitWasMadeButRunIsUnfinished(SleeperSystemTest sleeper) throws Exception { // Given - committer.setRunCommitterOnSend(false); + committer.setRunCommitterOnSend(sleeper, false); SystemTestStateStoreFakeCommits commitsDsl = sleeper.stateStore().fakeCommits(); commitsDsl.send(factory -> factory.addPartitionFile("root", "file.parquet", 100)); - committer.addRun(new StateStoreCommitterRun(Instant.now(), null, + committer.addRunClearQueue(new StateStoreCommitterRun(Instant.now(), null, List.of(new StateStoreCommitSummary(sleeper.tableProperties().get(TABLE_ID), "test-file-added", Instant.now())))); // When / Then diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index 66dc3c55eb..d2ce763f45 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -19,6 +19,7 @@ import sleeper.commit.StateStoreCommitter; import sleeper.configuration.properties.table.TablePropertiesProvider; import sleeper.core.statestore.StateStoreException; +import sleeper.systemtest.dsl.SleeperSystemTest; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; @@ -28,12 +29,15 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.stream.Stream; import static java.util.stream.Collectors.toUnmodifiableList; +import static sleeper.configuration.properties.table.TableProperty.TABLE_ID; public class InMemoryStateStoreCommitter { @@ -41,7 +45,7 @@ public class InMemoryStateStoreCommitter { private final InMemoryCompaction compaction; private final Queue queue = new LinkedList<>(); private final List runs = new ArrayList<>(); - private boolean runCommitterOnSend = true; + private final Map runCommitterOnSendByTableId = new HashMap<>(); public InMemoryStateStoreCommitter(InMemoryIngestByQueue ingest, InMemoryCompaction compaction) { this.ingest = ingest; @@ -52,19 +56,21 @@ public StateStoreCommitterDriver withContext(SystemTestContext context) { return new Driver(context); } - public void setRunCommitterOnSend(boolean runCommitterOnSend) { - this.runCommitterOnSend = runCommitterOnSend; + public void setRunCommitterOnSend(SleeperSystemTest sleeper, boolean runCommitterOnSend) { + runCommitterOnSendByTableId.put(sleeper.tableProperties().get(TABLE_ID), runCommitterOnSend); } - public void addRun(StateStoreCommitterRun run) { + public void addRunClearQueue(StateStoreCommitterRun run) { runs.add(run); + queue.clear(); } public class Driver implements StateStoreCommitterDriver { + private final SystemTestInstanceContext instance; private final StateStoreCommitter committer; private Driver(SystemTestContext context) { - SystemTestInstanceContext instance = context.instance(); + instance = context.instance(); TablePropertiesProvider tablePropertiesProvider = instance.getTablePropertiesProvider(); committer = new StateStoreCommitter(tablePropertiesProvider, compaction.jobStore(), ingest.jobStore(), @@ -75,7 +81,7 @@ private Driver(SystemTestContext context) { @Override public void sendCommitMessages(Stream messages) { messages.forEach(queue::add); - if (runCommitterOnSend) { + if (isRunCommitterOnSend()) { runCommitter(); } } @@ -103,6 +109,12 @@ private void runCommitter() { runs.add(new StateStoreCommitterRun(startTime, Instant.now(), commits)); } } + + private boolean isRunCommitterOnSend() { + return runCommitterOnSendByTableId.getOrDefault( + instance.getTableStatus().getTableUniqueId(), + true); + } } private static String failToLoadS3Object(String key) { From 1a51d62903283ad73ea866cf3ef98cd777be6025 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:55:56 +0000 Subject: [PATCH 23/49] Unit test decrementWaitForNumCommits --- .../statestore/StateStoreCommitterRun.java | 21 ++++ .../SystemTestStateStoreFakeCommits.java | 22 +--- ...teStoreCommitterRunWaitForCommitsTest.java | 119 ++++++++++++++++++ 3 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java index 79c17c55a3..56f96cfcbc 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java @@ -18,8 +18,12 @@ import java.time.Instant; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.summingInt; + public class StateStoreCommitterRun { private final Instant startTime; @@ -32,6 +36,23 @@ public StateStoreCommitterRun(Instant startTime, Instant finishTime, List runs, Map waitForNumCommitsByTableId) { + Map numCommitsByTableId = runs.stream() + .flatMap(run -> run.getCommits().stream()) + .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); + numCommitsByTableId.forEach((tableId, numCommits) -> { + waitForNumCommitsByTableId.compute(tableId, (id, count) -> { + if (count == null) { + return null; + } else if (numCommits >= count) { + return null; + } else { + return count - numCommits; + } + }); + }); + } + public Instant getStartTime() { return startTime; } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index 70f1ae5e2b..ef57ee4637 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -27,9 +27,6 @@ import java.util.function.Function; import java.util.stream.Stream; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.summingInt; - public class SystemTestStateStoreFakeCommits { private final SystemTestInstanceContext instance; @@ -56,7 +53,7 @@ public SystemTestStateStoreFakeCommits send(Function { List runs = driver.getRunsAfter(findCommitsFromTime); - decrementWaitForNumCommits(runs); + StateStoreCommitterRun.decrementWaitForNumCommits(runs, waitForNumCommitsByTableId); updateFindCommitsFromTime(runs); return waitForNumCommitsByTableId.isEmpty(); }); @@ -70,23 +67,6 @@ private void send(Stream messages) { (id, count) -> count == null ? 1 : count + 1))); } - private void decrementWaitForNumCommits(List runs) { - Map numCommitsByTableId = runs.stream() - .flatMap(run -> run.getCommits().stream()) - .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); - numCommitsByTableId.forEach((tableId, numCommits) -> { - waitForNumCommitsByTableId.compute(tableId, (id, count) -> { - if (count == null) { - return null; - } else if (numCommits >= count) { - return null; - } else { - return count - numCommits; - } - }); - }); - } - private void updateFindCommitsFromTime(List runs) { runs.stream() .map(StateStoreCommitterRun::getLastTime) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java new file mode 100644 index 0000000000..9acb008bf1 --- /dev/null +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java @@ -0,0 +1,119 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toUnmodifiableList; +import static org.assertj.core.api.Assertions.assertThat; + +public class StateStoreCommitterRunWaitForCommitsTest { + Map waitForNumCommitsByTableId = new HashMap<>(); + + @Test + void shouldFindOneCommitWasMadeAgainstCorrectTable() { + // Given + waitForNumCommitsByTableId.put("test-table", 2); + + // When + StateStoreCommitterRun.decrementWaitForNumCommits( + List.of(runWithCommitsToTable(1, "test-table")), + waitForNumCommitsByTableId); + + // Then + assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("test-table", 1)); + } + + @Test + void shouldFindAllCommitsWereMadeAgainstCorrectTable() { + // Given + waitForNumCommitsByTableId.put("test-table", 2); + + // When + StateStoreCommitterRun.decrementWaitForNumCommits( + List.of(runWithCommitsToTable(2, "test-table")), + waitForNumCommitsByTableId); + + // Then + assertThat(waitForNumCommitsByTableId).isEmpty(); + } + + @Test + void shouldFindCommitAgainstWrongTable() { + // Given + waitForNumCommitsByTableId.put("test-table", 2); + + // When + StateStoreCommitterRun.decrementWaitForNumCommits( + List.of(runWithCommitsToTable(1, "other-table")), + waitForNumCommitsByTableId); + + // Then + assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("test-table", 2)); + } + + @Test + void shouldFindCommitsAgainstMultipleTablesInOneRun() { + // Given + waitForNumCommitsByTableId.put("table-1", 2); + waitForNumCommitsByTableId.put("table-2", 2); + + // When + StateStoreCommitterRun.decrementWaitForNumCommits( + List.of(runWithCommits(List.of(commitToTable("table-1"), commitToTable("table-2")))), + waitForNumCommitsByTableId); + + // Then + assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("table-1", 1, "table-2", 1)); + } + + @Test + void shouldFindCommitsAgainstMultipleTablesInSeparateRuns() { + // Given + waitForNumCommitsByTableId.put("table-1", 2); + waitForNumCommitsByTableId.put("table-2", 2); + + // When + StateStoreCommitterRun.decrementWaitForNumCommits(List.of( + runWithCommitsToTable(1, "table-1"), + runWithCommitsToTable(1, "table-2")), + waitForNumCommitsByTableId); + + // Then + assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("table-1", 1, "table-2", 1)); + } + + private StateStoreCommitterRun runWithCommitsToTable(int commits, String tableId) { + return runWithCommits(IntStream.range(0, commits) + .mapToObj(i -> new StateStoreCommitSummary(tableId, "test-commit-type", Instant.now())) + .collect(toUnmodifiableList())); + } + + private StateStoreCommitterRun runWithCommits(List commits) { + return new StateStoreCommitterRun(Instant.now(), Instant.now(), commits); + } + + private StateStoreCommitSummary commitToTable(String tableId) { + return new StateStoreCommitSummary(tableId, "test-commit-type", Instant.now()); + } + +} From 1dbe54691c3e9a7f455489a934e1de883fd46d4a Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:18:19 +0000 Subject: [PATCH 24/49] Handle unknown run times --- .../StateStoreCommitterRunsBuilder.java | 16 +- .../StateStoreCommitterRunsBuilderTest.java | 21 +++ .../statestore/StateStoreCommitterRun.java | 21 ++- .../SystemTestStateStoreFakeCommits.java | 10 +- .../StateStoreCommitterRunLastTimeTest.java | 172 ++++++++++++++++++ 5 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java index e6d9684be0..c8571fc79e 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -73,12 +73,20 @@ void add(Object event) { lastRun = new LambdaRun((LambdaStarted) event); runs.add(lastRun); } else if (event instanceof LambdaFinished) { - lastRun.finished((LambdaFinished) event); + lastRunOrUnknownStartTime().finished((LambdaFinished) event); } else if (event instanceof StateStoreCommitSummary) { - lastRun.committed((StateStoreCommitSummary) event); + lastRunOrUnknownStartTime().committed((StateStoreCommitSummary) event); } } + private LambdaRun lastRunOrUnknownStartTime() { + if (lastRun == null) { + lastRun = new LambdaRun(); + runs.add(lastRun); + } + return lastRun; + } + Stream runs() { return runs.stream(); } @@ -89,6 +97,10 @@ private static class LambdaRun { private Instant finishTime; private List commits = new ArrayList<>(); + LambdaRun() { + startTime = null; + } + LambdaRun(LambdaStarted event) { this.startTime = event.getStartTime(); } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index e6bfb1944f..e371f6d934 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -89,6 +89,27 @@ void shouldBuildUnfinishedRunWithOneCommit() { List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } + @Test + void shouldBuildRunWithOneCommitAndNoStartMessage() { + // Given + add("test-logstream", "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(null, null, + List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); + } + + @Test + void shouldBuildRunWithOnlyFinishMessage() { + // Given + add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:30Z (ran for 1 minute)"); + + // When / Then + assertThat(builder.buildRuns()).containsExactly( + new StateStoreCommitterRun(null, Instant.parse("2024-08-13T12:13:30Z"), List.of())); + } + @Test void shouldIgnoreUnrecognisedLog() { // Given diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java index 56f96cfcbc..6d3e0a0760 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summingInt; @@ -53,6 +54,12 @@ public static void decrementWaitForNumCommits(List runs, }); } + public static Optional getLastTime(List runs) { + return runs.stream() + .flatMap(run -> run.getLastTime().stream()) + .max(Comparator.naturalOrder()); + } + public Instant getStartTime() { return startTime; } @@ -61,14 +68,18 @@ public Instant getFinishTime() { return finishTime; } - public Instant getLastTime() { + public Optional getLastTime() { if (finishTime != null) { - return finishTime; + return Optional.of(finishTime); } - return commits.stream() + Optional commitTime = commits.stream() .map(StateStoreCommitSummary::getFinishTime) - .max(Comparator.naturalOrder()) - .orElse(startTime); + .max(Comparator.naturalOrder()); + if (commitTime.isPresent()) { + return commitTime; + } else { + return Optional.ofNullable(startTime); + } } public List getCommits() { diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index ef57ee4637..a3faf912bd 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -20,7 +20,6 @@ import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import java.time.Instant; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -54,7 +53,7 @@ public SystemTestStateStoreFakeCommits waitForCommits(PollWithRetries poll) thro poll.pollUntil("all state store commits are applied", () -> { List runs = driver.getRunsAfter(findCommitsFromTime); StateStoreCommitterRun.decrementWaitForNumCommits(runs, waitForNumCommitsByTableId); - updateFindCommitsFromTime(runs); + StateStoreCommitterRun.getLastTime(runs).ifPresent(lastTime -> findCommitsFromTime = lastTime); return waitForNumCommitsByTableId.isEmpty(); }); return this; @@ -67,13 +66,6 @@ private void send(Stream messages) { (id, count) -> count == null ? 1 : count + 1))); } - private void updateFindCommitsFromTime(List runs) { - runs.stream() - .map(StateStoreCommitterRun::getLastTime) - .max(Comparator.naturalOrder()) - .ifPresent(lastTime -> findCommitsFromTime = lastTime); - } - private StateStoreCommitMessageFactory messageFactory() { return new StateStoreCommitMessageFactory(instance.getTableStatus().getTableUniqueId()); } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java new file mode 100644 index 0000000000..a5b7a846c7 --- /dev/null +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java @@ -0,0 +1,172 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StateStoreCommitterRunLastTimeTest { + + @Test + void shouldGetStartTimeWhenRunJustStarted() { + // Given + Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); + List runs = List.of(unfinishedRun(startTime)); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(startTime); + } + + @Test + void shouldGetFinishTimeWhenRunFinished() { + // Given + Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); + Instant finishTime = Instant.parse("2024-08-14T09:58:10Z"); + List runs = List.of(finishedRun(startTime, finishTime)); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(finishTime); + } + + @Test + void shouldGetFinishTimeWhenRunFinishedWithACommit() { + // Given + Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); + Instant commitTime = Instant.parse("2024-08-14T09:58:05Z"); + Instant finishTime = Instant.parse("2024-08-14T09:58:10Z"); + List runs = List.of(finishedRun(startTime, finishTime, commitAtTime(commitTime))); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(finishTime); + } + + @Test + void shouldGetCommitTimeWhenRunUnfinishedWithACommit() { + // Given + Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); + Instant commitTime = Instant.parse("2024-08-14T09:58:05Z"); + List runs = List.of(unfinishedRun(startTime, commitAtTime(commitTime))); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(commitTime); + } + + @Test + void shouldGetLatestRunWhenLastInList() { + // Given + Instant runTime1 = Instant.parse("2024-08-14T09:58:00Z"); + Instant runTime2 = Instant.parse("2024-08-14T09:59:00Z"); + List runs = List.of(unfinishedRun(runTime1), unfinishedRun(runTime2)); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(runTime2); + } + + @Test + void shouldGetLatestRunWhenFirstInList() { + // Given + Instant runTime1 = Instant.parse("2024-08-14T09:58:00Z"); + Instant runTime2 = Instant.parse("2024-08-14T09:59:00Z"); + List runs = List.of(unfinishedRun(runTime2), unfinishedRun(runTime1)); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(runTime2); + } + + @Test + void shouldGetLatestCommitWhenLastInList() { + // Given + Instant runTime = Instant.parse("2024-08-14T09:58:00Z"); + Instant commitTime1 = Instant.parse("2024-08-14T09:58:01Z"); + Instant commitTime2 = Instant.parse("2024-08-14T09:58:02Z"); + List runs = List.of( + unfinishedRun(runTime, commitAtTime(commitTime1), commitAtTime(commitTime2))); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(commitTime2); + } + + @Test + void shouldGetLatestCommitWhenFirstInList() { + // Given + Instant runTime = Instant.parse("2024-08-14T09:58:00Z"); + Instant commitTime1 = Instant.parse("2024-08-14T09:58:01Z"); + Instant commitTime2 = Instant.parse("2024-08-14T09:58:02Z"); + List runs = List.of( + unfinishedRun(runTime, commitAtTime(commitTime2), commitAtTime(commitTime1))); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(commitTime2); + } + + @Test + void shouldGetCommitTimeWhenRunStartTimeUnknown() { + // Given + Instant commitTime = Instant.parse("2024-08-14T09:58:01Z"); + List runs = List.of( + unknownStartUnfinishedRun(commitAtTime(commitTime))); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)) + .contains(commitTime); + } + + @Test + void shouldGetNoTimeWhenNoRuns() { + // Given + List runs = List.of(); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)).isEmpty(); + } + + @Test + void shouldGetNoTimeWhenRunWithUnknownTime() { + // Given + List runs = List.of(unknownStartUnfinishedRun()); + + // When / Then + assertThat(StateStoreCommitterRun.getLastTime(runs)).isEmpty(); + } + + private StateStoreCommitterRun unfinishedRun(Instant startTime, StateStoreCommitSummary... commits) { + return new StateStoreCommitterRun(startTime, null, List.of(commits)); + } + + private StateStoreCommitterRun finishedRun(Instant startTime, Instant finishTime, StateStoreCommitSummary... commits) { + return new StateStoreCommitterRun(startTime, finishTime, List.of(commits)); + } + + private StateStoreCommitterRun unknownStartUnfinishedRun(StateStoreCommitSummary... commits) { + return new StateStoreCommitterRun(null, null, List.of(commits)); + } + + private StateStoreCommitSummary commitAtTime(Instant time) { + return new StateStoreCommitSummary("test-table", "test-commit-type", time); + } +} From 197f831bccbe54e8d91b40aa5609cedf6658bb56 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:29:34 +0000 Subject: [PATCH 25/49] Add slack & logging in query for committer runs --- .../SystemTestStateStoreFakeCommits.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index a3faf912bd..d406e35315 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -15,10 +15,14 @@ */ package sleeper.systemtest.dsl.statestore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; +import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Map; @@ -27,16 +31,19 @@ import java.util.stream.Stream; public class SystemTestStateStoreFakeCommits { + public static final Logger LOGGER = LoggerFactory.getLogger(SystemTestStateStoreFakeCommits.class); + + private static final Duration QUERY_RUNS_SINCE_AGE = Duration.ofMinutes(1); private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; private final Map waitForNumCommitsByTableId = new ConcurrentHashMap<>(); - private Instant findCommitsFromTime; + private Instant getRunsAfterTime; public SystemTestStateStoreFakeCommits(SystemTestContext context) { instance = context.instance(); driver = context.instance().adminDrivers().stateStoreCommitter(context); - findCommitsFromTime = context.reporting().getRecordingStartTime(); + getRunsAfterTime = context.reporting().getRecordingStartTime().minus(QUERY_RUNS_SINCE_AGE); } public SystemTestStateStoreFakeCommits sendBatched(Function> buildCommits) { @@ -50,10 +57,12 @@ public SystemTestStateStoreFakeCommits send(Function { - List runs = driver.getRunsAfter(findCommitsFromTime); + List runs = driver.getRunsAfter(getRunsAfterTime); StateStoreCommitterRun.decrementWaitForNumCommits(runs, waitForNumCommitsByTableId); - StateStoreCommitterRun.getLastTime(runs).ifPresent(lastTime -> findCommitsFromTime = lastTime); + StateStoreCommitterRun.getLastTime(runs).ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_SINCE_AGE)); + LOGGER.info("Remaining unapplied commits by table ID: {}", waitForNumCommitsByTableId); return waitForNumCommitsByTableId.isEmpty(); }); return this; From f35b7cb238569e9ec3501e2712d12c7545370371 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:32:32 +0000 Subject: [PATCH 26/49] Grant reading state store committer logs --- .../java/sleeper/cdk/stack/StateStoreCommitterStack.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java index d94bcdefd3..22afd17678 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java @@ -73,7 +73,7 @@ public StateStoreCommitterStack( LambdaCode committerJar = jars.lambdaCode(BuiltJar.STATESTORE, jarsBucket); commitQueue = sqsQueueForStateStoreCommitter(policiesStack, topic, errorMetrics); - lambdaToCommitStateStoreUpdates(committerJar, + lambdaToCommitStateStoreUpdates(policiesStack, committerJar, configBucketStack, tableIndexStack, stateStoreStacks, compactionStatusStore, ingestStatusStore); } @@ -110,8 +110,8 @@ private Queue sqsQueueForStateStoreCommitter(ManagedPoliciesStack policiesStack, } private void lambdaToCommitStateStoreUpdates( - LambdaCode committerJar, ConfigBucketStack configBucketStack, TableIndexStack tableIndexStack, - StateStoreStacks stateStoreStacks, + ManagedPoliciesStack policiesStack, LambdaCode committerJar, + ConfigBucketStack configBucketStack, TableIndexStack tableIndexStack, StateStoreStacks stateStoreStacks, CompactionStatusStoreResources compactionStatusStore, IngestStatusStoreResources ingestStatusStore) { Map environmentVariables = Utils.createDefaultEnvironment(instanceProperties); @@ -135,6 +135,7 @@ private void lambdaToCommitStateStoreUpdates( .batchSize(instanceProperties.getInt(STATESTORE_COMMITTER_BATCH_SIZE)) .build()); + logGroup.grantRead(policiesStack.getReportingPolicyForGrants()); configBucketStack.grantRead(handlerFunction); tableIndexStack.grantRead(handlerFunction); stateStoreStacks.grantReadWriteAllFilesAndPartitions(handlerFunction); From 8c1cdc072cc836fe64e75f662947a6a119898ee5 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:38:14 +0000 Subject: [PATCH 27/49] Apply query time slack to end time --- .../drivers/statestore/AwsStateStoreCommitterDriver.java | 4 ++-- .../dsl/statestore/StateStoreCommitterDriver.java | 2 +- .../dsl/statestore/SystemTestStateStoreFakeCommits.java | 8 ++++---- .../dsl/testutil/drivers/InMemoryStateStoreCommitter.java | 3 ++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java index 2c8f621270..32132374ff 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java @@ -61,13 +61,13 @@ public void sendCommitMessages(Stream messages) { } @Override - public List getRunsAfter(Instant startTime) { + public List getRunsInPeriod(Instant startTime, Instant endTime) { String logGroupName = instance.getInstanceProperties().get(STATESTORE_COMMITTER_LOG_GROUP); LOGGER.info("Submitting logs query for log group {} starting at time {}", logGroupName, startTime); String queryId = cloudWatch.startQuery(builder -> builder .logGroupName(logGroupName) .startTime(startTime.getEpochSecond()) - .endTime(Instant.now().plus(Duration.ofMinutes(1)).getEpochSecond()) + .endTime(endTime.getEpochSecond()) .limit(10000) .queryString("fields @timestamp, @message, @logStream " + "| filter @message like /Lambda (started|finished) at|Applied request to table/ " + diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java index 81ad7ee0fa..705c929500 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java @@ -23,5 +23,5 @@ public interface StateStoreCommitterDriver { void sendCommitMessages(Stream messages); - List getRunsAfter(Instant startTime); + List getRunsInPeriod(Instant startTime, Instant endTime); } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index d406e35315..a8df1d2445 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -33,7 +33,7 @@ public class SystemTestStateStoreFakeCommits { public static final Logger LOGGER = LoggerFactory.getLogger(SystemTestStateStoreFakeCommits.class); - private static final Duration QUERY_RUNS_SINCE_AGE = Duration.ofMinutes(1); + private static final Duration QUERY_RUNS_TIME_SLACK = Duration.ofMinutes(1); private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; @@ -43,7 +43,7 @@ public class SystemTestStateStoreFakeCommits { public SystemTestStateStoreFakeCommits(SystemTestContext context) { instance = context.instance(); driver = context.instance().adminDrivers().stateStoreCommitter(context); - getRunsAfterTime = context.reporting().getRecordingStartTime().minus(QUERY_RUNS_SINCE_AGE); + getRunsAfterTime = context.reporting().getRecordingStartTime().minus(QUERY_RUNS_TIME_SLACK); } public SystemTestStateStoreFakeCommits sendBatched(Function> buildCommits) { @@ -59,9 +59,9 @@ public SystemTestStateStoreFakeCommits send(Function { - List runs = driver.getRunsAfter(getRunsAfterTime); + List runs = driver.getRunsInPeriod(getRunsAfterTime, Instant.now().plus(QUERY_RUNS_TIME_SLACK)); StateStoreCommitterRun.decrementWaitForNumCommits(runs, waitForNumCommitsByTableId); - StateStoreCommitterRun.getLastTime(runs).ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_SINCE_AGE)); + StateStoreCommitterRun.getLastTime(runs).ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_TIME_SLACK)); LOGGER.info("Remaining unapplied commits by table ID: {}", waitForNumCommitsByTableId); return waitForNumCommitsByTableId.isEmpty(); }); diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index d2ce763f45..7fa391688f 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -87,9 +87,10 @@ public void sendCommitMessages(Stream messages) { } @Override - public List getRunsAfter(Instant startTime) { + public List getRunsInPeriod(Instant startTime, Instant endTime) { return runs.stream() .filter(run -> run.getStartTime().compareTo(startTime) >= 0) + .filter(run -> run.getStartTime().compareTo(endTime) <= 0) .collect(toUnmodifiableList()); } From 59cfead12e7a21c6980592f7f44a6d12062d7f5d Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:40:05 +0000 Subject: [PATCH 28/49] Reduce slack for quering committer runs --- .../dsl/statestore/SystemTestStateStoreFakeCommits.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index a8df1d2445..40f703a42f 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -33,7 +33,7 @@ public class SystemTestStateStoreFakeCommits { public static final Logger LOGGER = LoggerFactory.getLogger(SystemTestStateStoreFakeCommits.class); - private static final Duration QUERY_RUNS_TIME_SLACK = Duration.ofMinutes(1); + private static final Duration QUERY_RUNS_TIME_SLACK = Duration.ofSeconds(5); private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; From b3bdc33eec3966dc360a1163473be666ac81d128 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:26:54 +0000 Subject: [PATCH 29/49] Introduce StateStoreCommitterLogEntry --- .../StateStoreCommitterLogEntry.java | 4 +- .../StateStoreCommitterRunsBuilder.java | 17 ++--- .../StateStoreCommitterLogEntryTest.java | 10 +-- .../StateStoreCommitterRunsBuilderTest.java | 26 +++---- .../statestore/StateStoreCommitSummary.java | 22 ++++-- .../statestore/StateStoreCommitterDriver.java | 8 +++ .../StateStoreCommitterLogEntry.java | 34 +++++++++ .../statestore/StateStoreCommitterRun.java | 55 ++++---------- .../StateStoreCommitterRunFinished.java | 65 +++++++++++++++++ .../StateStoreCommitterRunStarted.java | 65 +++++++++++++++++ .../StateStoreCommitterRunsBuilder.java | 20 ++++++ .../SystemTestStateStoreFakeCommits.java | 10 ++- .../statestore/WaitForStateStoreCommits.java | 46 ++++++++++++ ...teStoreCommitterLogEntryLastTimeTest.java} | 71 ++++++++----------- .../SystemTestStateStoreFakeCommitsTest.java | 4 +- ...java => WaitForStateStoreCommitsTest.java} | 38 ++++------ .../drivers/InMemoryStateStoreCommitter.java | 4 +- 17 files changed, 349 insertions(+), 150 deletions(-) create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunFinished.java create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunStarted.java create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java rename java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/{StateStoreCommitterRunLastTimeTest.java => StateStoreCommitterLogEntryLastTimeTest.java} (57%) rename java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/{StateStoreCommitterRunWaitForCommitsTest.java => WaitForStateStoreCommitsTest.java} (64%) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java index 4e9101b665..407a15e08c 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java @@ -43,7 +43,7 @@ private CapturingGroups() { private StateStoreCommitterLogEntry() { } - public static Object readEvent(String message) { + public static Object readEvent(String logStream, String message) { Matcher matcher = MESSAGE_PATTERN.matcher(message); if (!matcher.find()) { return null; @@ -63,7 +63,7 @@ public static Object readEvent(String message) { if (tableId != null) { String type = matcher.group(CapturingGroups.TYPE); String commitTime = matcher.group(CapturingGroups.COMMIT_TIME); - return new StateStoreCommitSummary(tableId, type, Instant.parse(commitTime)); + return new StateStoreCommitSummary(logStream, tableId, type, Instant.parse(commitTime)); } return null; } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java index c8571fc79e..425ed5264a 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -37,12 +37,12 @@ public class StateStoreCommitterRunsBuilder { private final Map logStreamByName = new LinkedHashMap<>(); public void add(List entry) { - LogStream logStream = null; + String logStream = null; String message = null; for (ResultField field : entry) { switch (field.field()) { case "@logStream": - logStream = logStreamByName.computeIfAbsent(field.value(), name -> new LogStream()); + logStream = field.value(); break; case "@message": message = field.value(); @@ -53,13 +53,14 @@ public void add(List entry) { } Objects.requireNonNull(logStream, "Log stream not found"); Objects.requireNonNull(message, "Log message not found"); - logStream.add(StateStoreCommitterLogEntry.readEvent(message)); + logStreamByName.computeIfAbsent(logStream, name -> new LogStream()) + .add(StateStoreCommitterLogEntry.readEvent(logStream, message)); } public List buildRuns() { - return logStreamByName.values().stream() - .flatMap(LogStream::runs) - .map(LambdaRun::build) + return logStreamByName.entrySet().stream() + .flatMap(entry -> entry.getValue().runs() + .map(run -> run.build(entry.getKey()))) .collect(toUnmodifiableList()); } @@ -113,8 +114,8 @@ void committed(StateStoreCommitSummary commit) { commits.add(commit); } - StateStoreCommitterRun build() { - return new StateStoreCommitterRun(startTime, finishTime, commits); + StateStoreCommitterRun build(String logStream) { + return new StateStoreCommitterRun(logStream, startTime, finishTime, commits); } } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java index e11fc03548..afa6fc567a 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java @@ -33,7 +33,7 @@ void shouldReadLambdaStarted() { String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( + assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isEqualTo( new LambdaStarted(Instant.parse("2024-08-13T12:12:00Z"))); } @@ -43,7 +43,7 @@ void shouldReadLambdaFinished() { String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( + assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isEqualTo( new LambdaFinished(Instant.parse("2024-08-13T12:13:00Z"))); } @@ -53,8 +53,8 @@ void shouldReadCommitApplied() { String message = "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent(message)).isEqualTo( - new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))); + assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isEqualTo( + new StateStoreCommitSummary("test-stream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))); } @Test @@ -63,6 +63,6 @@ void shouldReadUnrecognisedLog() { String message = "some other log"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent(message)).isNull(); + assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isNull(); } } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java index e371f6d934..8abc7158f5 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java @@ -37,7 +37,7 @@ void shouldBuildSingleRunNoCommits() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); + new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); } @Test @@ -50,8 +50,8 @@ void shouldBuildOverlappingRunsOnTwoLogStreams() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of()), - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:30Z"), Instant.parse("2024-08-13T12:13:30Z"), List.of())); + new StateStoreCommitterRun("stream-1", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of()), + new StateStoreCommitterRun("stream-2", Instant.parse("2024-08-13T12:12:30Z"), Instant.parse("2024-08-13T12:13:30Z"), List.of())); } @Test @@ -63,8 +63,8 @@ void shouldBuildSingleRunWithOneCommit() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), - List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); + new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), + List.of(new StateStoreCommitSummary("test-logstream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } @Test @@ -74,7 +74,7 @@ void shouldBuildUnfinishedRun() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), null, List.of())); + new StateStoreCommitterRun("test-stream", Instant.parse("2024-08-13T12:12:00Z"), null, List.of())); } @Test @@ -85,8 +85,8 @@ void shouldBuildUnfinishedRunWithOneCommit() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), null, - List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); + new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), null, + List.of(new StateStoreCommitSummary("test-logstream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } @Test @@ -96,8 +96,8 @@ void shouldBuildRunWithOneCommitAndNoStartMessage() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(null, null, - List.of(new StateStoreCommitSummary("test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); + new StateStoreCommitterRun("test-logstream", null, null, + List.of(new StateStoreCommitSummary("test-logstream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); } @Test @@ -107,7 +107,7 @@ void shouldBuildRunWithOnlyFinishMessage() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(null, Instant.parse("2024-08-13T12:13:30Z"), List.of())); + new StateStoreCommitterRun("test-logstream", null, Instant.parse("2024-08-13T12:13:30Z"), List.of())); } @Test @@ -127,7 +127,7 @@ void shouldReadLogWithFieldsInReverseOrder() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); + new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); } @Test @@ -144,7 +144,7 @@ void shouldIgnoreExtraFieldsInLogMessage() { // When / Then assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun(Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); + new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); } void add(String logStream, String message) { diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java index 844559da80..f511fb23c2 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitSummary.java @@ -18,12 +18,14 @@ import java.time.Instant; import java.util.Objects; -public class StateStoreCommitSummary { +public class StateStoreCommitSummary implements StateStoreCommitterLogEntry { + private final String logStream; private final String tableId; private final String type; private final Instant finishTime; - public StateStoreCommitSummary(String tableId, String type, Instant finishTime) { + public StateStoreCommitSummary(String logStream, String tableId, String type, Instant finishTime) { + this.logStream = logStream; this.tableId = tableId; this.type = type; this.finishTime = finishTime; @@ -41,9 +43,19 @@ public Instant getFinishTime() { return finishTime; } + @Override + public String getLogStream() { + return logStream; + } + + @Override + public Instant getTimeInCommitter() { + return finishTime; + } + @Override public int hashCode() { - return Objects.hash(tableId, type, finishTime); + return Objects.hash(logStream, tableId, type, finishTime); } @Override @@ -55,11 +67,11 @@ public boolean equals(Object obj) { return false; } StateStoreCommitSummary other = (StateStoreCommitSummary) obj; - return Objects.equals(tableId, other.tableId) && Objects.equals(type, other.type) && Objects.equals(finishTime, other.finishTime); + return Objects.equals(logStream, other.logStream) && Objects.equals(tableId, other.tableId) && Objects.equals(type, other.type) && Objects.equals(finishTime, other.finishTime); } @Override public String toString() { - return "StateStoreCommitSummary{tableId=" + tableId + ", type=" + type + ", finishTime=" + finishTime + "}"; + return "StateStoreCommitSummary{logStream=" + logStream + ", tableId=" + tableId + ", type=" + type + ", finishTime=" + finishTime + "}"; } } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java index 705c929500..801f5976c1 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java @@ -19,9 +19,17 @@ import java.util.List; import java.util.stream.Stream; +import static java.util.stream.Collectors.toUnmodifiableList; + public interface StateStoreCommitterDriver { void sendCommitMessages(Stream messages); List getRunsInPeriod(Instant startTime, Instant endTime); + + default List getLogsInPeriod(Instant startTime, Instant endTime) { + return getRunsInPeriod(startTime, endTime) + .stream().flatMap(run -> run.logs()) + .collect(toUnmodifiableList()); + } } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java new file mode 100644 index 0000000000..83de134f08 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +public interface StateStoreCommitterLogEntry { + + String getLogStream(); + + Instant getTimeInCommitter(); + + static Optional getLastTime(List entries) { + return entries.stream() + .map(StateStoreCommitterLogEntry::getTimeInCommitter) + .max(Comparator.naturalOrder()); + } +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java index 6d3e0a0760..6b31dd61ca 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java @@ -16,50 +16,25 @@ package sleeper.systemtest.dsl.statestore; import java.time.Instant; -import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; - -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.summingInt; +import java.util.stream.Stream; public class StateStoreCommitterRun { + private final String logStream; private final Instant startTime; private final Instant finishTime; private final List commits; - public StateStoreCommitterRun(Instant startTime, Instant finishTime, List commits) { + public StateStoreCommitterRun(String logStream, Instant startTime, Instant finishTime, List commits) { + this.logStream = logStream; this.startTime = startTime; this.finishTime = finishTime; this.commits = commits; } - public static void decrementWaitForNumCommits(List runs, Map waitForNumCommitsByTableId) { - Map numCommitsByTableId = runs.stream() - .flatMap(run -> run.getCommits().stream()) - .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); - numCommitsByTableId.forEach((tableId, numCommits) -> { - waitForNumCommitsByTableId.compute(tableId, (id, count) -> { - if (count == null) { - return null; - } else if (numCommits >= count) { - return null; - } else { - return count - numCommits; - } - }); - }); - } - - public static Optional getLastTime(List runs) { - return runs.stream() - .flatMap(run -> run.getLastTime().stream()) - .max(Comparator.naturalOrder()); - } - public Instant getStartTime() { return startTime; } @@ -68,24 +43,18 @@ public Instant getFinishTime() { return finishTime; } - public Optional getLastTime() { - if (finishTime != null) { - return Optional.of(finishTime); - } - Optional commitTime = commits.stream() - .map(StateStoreCommitSummary::getFinishTime) - .max(Comparator.naturalOrder()); - if (commitTime.isPresent()) { - return commitTime; - } else { - return Optional.ofNullable(startTime); - } - } - public List getCommits() { return commits; } + public Stream logs() { + return Stream.of( + Optional.ofNullable(startTime).map(time -> new StateStoreCommitterRunStarted(logStream, time)).stream(), + commits.stream(), + Optional.ofNullable(finishTime).map(time -> new StateStoreCommitterRunFinished(logStream, time)).stream()) + .flatMap(s -> s); + } + @Override public int hashCode() { return Objects.hash(startTime, finishTime, commits); diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunFinished.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunFinished.java new file mode 100644 index 0000000000..1a060ddca6 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunFinished.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.time.Instant; +import java.util.Objects; + +public class StateStoreCommitterRunFinished implements StateStoreCommitterLogEntry { + private final String logStream; + private final Instant finishTime; + + public StateStoreCommitterRunFinished(String logStream, Instant finishTime) { + this.logStream = logStream; + this.finishTime = finishTime; + } + + @Override + public String getLogStream() { + return logStream; + } + + public Instant getFinishTime() { + return finishTime; + } + + @Override + public Instant getTimeInCommitter() { + return finishTime; + } + + @Override + public int hashCode() { + return Objects.hash(logStream, finishTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StateStoreCommitterRunFinished)) { + return false; + } + StateStoreCommitterRunFinished other = (StateStoreCommitterRunFinished) obj; + return Objects.equals(logStream, other.logStream) && Objects.equals(finishTime, other.finishTime); + } + + @Override + public String toString() { + return "StateStoreCommitterRunFinished{logStream=" + logStream + ", finishTime=" + finishTime + "}"; + } +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunStarted.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunStarted.java new file mode 100644 index 0000000000..69c76a6c61 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunStarted.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.time.Instant; +import java.util.Objects; + +public class StateStoreCommitterRunStarted implements StateStoreCommitterLogEntry { + private final String logStream; + private final Instant startTime; + + public StateStoreCommitterRunStarted(String logStream, Instant startTime) { + this.logStream = logStream; + this.startTime = startTime; + } + + @Override + public String getLogStream() { + return logStream; + } + + public Instant getStartTime() { + return startTime; + } + + @Override + public Instant getTimeInCommitter() { + return startTime; + } + + @Override + public int hashCode() { + return Objects.hash(logStream, startTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StateStoreCommitterRunStarted)) { + return false; + } + StateStoreCommitterRunStarted other = (StateStoreCommitterRunStarted) obj; + return Objects.equals(logStream, other.logStream) && Objects.equals(startTime, other.startTime); + } + + @Override + public String toString() { + return "StateStoreCommitterRunStarted{logStream=" + logStream + ", startTime=" + startTime + "}"; + } +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java new file mode 100644 index 0000000000..649c46835c --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java @@ -0,0 +1,20 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +public class StateStoreCommitterRunsBuilder { + +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index 40f703a42f..c8d85a06cc 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -24,6 +24,7 @@ import java.time.Duration; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -59,9 +60,12 @@ public SystemTestStateStoreFakeCommits send(Function { - List runs = driver.getRunsInPeriod(getRunsAfterTime, Instant.now().plus(QUERY_RUNS_TIME_SLACK)); - StateStoreCommitterRun.decrementWaitForNumCommits(runs, waitForNumCommitsByTableId); - StateStoreCommitterRun.getLastTime(runs).ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_TIME_SLACK)); + List logs = driver.getLogsInPeriod(getRunsAfterTime, Instant.now().plus(QUERY_RUNS_TIME_SLACK)); + WaitForStateStoreCommits.decrementWaitForNumCommits(logs, waitForNumCommitsByTableId); + logs.stream() + .map(StateStoreCommitterLogEntry::getTimeInCommitter) + .max(Comparator.naturalOrder()) + .ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_TIME_SLACK)); LOGGER.info("Remaining unapplied commits by table ID: {}", waitForNumCommitsByTableId); return waitForNumCommitsByTableId.isEmpty(); }); diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java new file mode 100644 index 0000000000..2fb6ec9449 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java @@ -0,0 +1,46 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.util.List; +import java.util.Map; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.summingInt; + +public class WaitForStateStoreCommits { + + private WaitForStateStoreCommits() { + } + + public static void decrementWaitForNumCommits(List entries, Map waitForNumCommitsByTableId) { + Map numCommitsByTableId = entries.stream() + .filter(entry -> entry instanceof StateStoreCommitSummary) + .map(entry -> (StateStoreCommitSummary) entry) + .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); + numCommitsByTableId.forEach((tableId, numCommits) -> { + waitForNumCommitsByTableId.compute(tableId, (id, count) -> { + if (count == null) { + return null; + } else if (numCommits >= count) { + return null; + } else { + return count - numCommits; + } + }); + }); + } +} diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java similarity index 57% rename from java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java rename to java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java index a5b7a846c7..f883d442bc 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunLastTimeTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java @@ -22,16 +22,16 @@ import static org.assertj.core.api.Assertions.assertThat; -public class StateStoreCommitterRunLastTimeTest { +public class StateStoreCommitterLogEntryLastTimeTest { @Test void shouldGetStartTimeWhenRunJustStarted() { // Given Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); - List runs = List.of(unfinishedRun(startTime)); + List entries = List.of(runStarted(startTime)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(startTime); } @@ -40,10 +40,10 @@ void shouldGetFinishTimeWhenRunFinished() { // Given Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); Instant finishTime = Instant.parse("2024-08-14T09:58:10Z"); - List runs = List.of(finishedRun(startTime, finishTime)); + List entries = List.of(runStarted(startTime), runFinished(finishTime)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(finishTime); } @@ -53,10 +53,11 @@ void shouldGetFinishTimeWhenRunFinishedWithACommit() { Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); Instant commitTime = Instant.parse("2024-08-14T09:58:05Z"); Instant finishTime = Instant.parse("2024-08-14T09:58:10Z"); - List runs = List.of(finishedRun(startTime, finishTime, commitAtTime(commitTime))); + List entries = List.of( + runStarted(startTime), commitAtTime(commitTime), runFinished(finishTime)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(finishTime); } @@ -65,10 +66,10 @@ void shouldGetCommitTimeWhenRunUnfinishedWithACommit() { // Given Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); Instant commitTime = Instant.parse("2024-08-14T09:58:05Z"); - List runs = List.of(unfinishedRun(startTime, commitAtTime(commitTime))); + List entries = List.of(runStarted(startTime), commitAtTime(commitTime)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(commitTime); } @@ -77,10 +78,10 @@ void shouldGetLatestRunWhenLastInList() { // Given Instant runTime1 = Instant.parse("2024-08-14T09:58:00Z"); Instant runTime2 = Instant.parse("2024-08-14T09:59:00Z"); - List runs = List.of(unfinishedRun(runTime1), unfinishedRun(runTime2)); + List entries = List.of(runStarted(runTime1), runStarted(runTime2)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(runTime2); } @@ -89,10 +90,10 @@ void shouldGetLatestRunWhenFirstInList() { // Given Instant runTime1 = Instant.parse("2024-08-14T09:58:00Z"); Instant runTime2 = Instant.parse("2024-08-14T09:59:00Z"); - List runs = List.of(unfinishedRun(runTime2), unfinishedRun(runTime1)); + List entries = List.of(runStarted(runTime2), runStarted(runTime1)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(runTime2); } @@ -102,11 +103,11 @@ void shouldGetLatestCommitWhenLastInList() { Instant runTime = Instant.parse("2024-08-14T09:58:00Z"); Instant commitTime1 = Instant.parse("2024-08-14T09:58:01Z"); Instant commitTime2 = Instant.parse("2024-08-14T09:58:02Z"); - List runs = List.of( - unfinishedRun(runTime, commitAtTime(commitTime1), commitAtTime(commitTime2))); + List entries = List.of( + runStarted(runTime), commitAtTime(commitTime1), commitAtTime(commitTime2)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(commitTime2); } @@ -116,11 +117,11 @@ void shouldGetLatestCommitWhenFirstInList() { Instant runTime = Instant.parse("2024-08-14T09:58:00Z"); Instant commitTime1 = Instant.parse("2024-08-14T09:58:01Z"); Instant commitTime2 = Instant.parse("2024-08-14T09:58:02Z"); - List runs = List.of( - unfinishedRun(runTime, commitAtTime(commitTime2), commitAtTime(commitTime1))); + List entries = List.of( + runStarted(runTime), commitAtTime(commitTime2), commitAtTime(commitTime1)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(commitTime2); } @@ -128,45 +129,31 @@ void shouldGetLatestCommitWhenFirstInList() { void shouldGetCommitTimeWhenRunStartTimeUnknown() { // Given Instant commitTime = Instant.parse("2024-08-14T09:58:01Z"); - List runs = List.of( - unknownStartUnfinishedRun(commitAtTime(commitTime))); + List entries = List.of(commitAtTime(commitTime)); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)) + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) .contains(commitTime); } @Test void shouldGetNoTimeWhenNoRuns() { // Given - List runs = List.of(); + List entries = List.of(); // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)).isEmpty(); + assertThat(StateStoreCommitterLogEntry.getLastTime(entries)).isEmpty(); } - @Test - void shouldGetNoTimeWhenRunWithUnknownTime() { - // Given - List runs = List.of(unknownStartUnfinishedRun()); - - // When / Then - assertThat(StateStoreCommitterRun.getLastTime(runs)).isEmpty(); - } - - private StateStoreCommitterRun unfinishedRun(Instant startTime, StateStoreCommitSummary... commits) { - return new StateStoreCommitterRun(startTime, null, List.of(commits)); - } - - private StateStoreCommitterRun finishedRun(Instant startTime, Instant finishTime, StateStoreCommitSummary... commits) { - return new StateStoreCommitterRun(startTime, finishTime, List.of(commits)); + private StateStoreCommitterRunStarted runStarted(Instant time) { + return new StateStoreCommitterRunStarted("test-stream", time); } - private StateStoreCommitterRun unknownStartUnfinishedRun(StateStoreCommitSummary... commits) { - return new StateStoreCommitterRun(null, null, List.of(commits)); + private StateStoreCommitterRunFinished runFinished(Instant time) { + return new StateStoreCommitterRunFinished("test-stream", time); } private StateStoreCommitSummary commitAtTime(Instant time) { - return new StateStoreCommitSummary("test-table", "test-commit-type", time); + return new StateStoreCommitSummary("test-stream", "test-table", "test-commit-type", time); } } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index 44d501ab75..11e3433ea3 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -78,8 +78,8 @@ void shouldWaitForCommitWhenCommitWasMadeButRunIsUnfinished(SleeperSystemTest sl committer.setRunCommitterOnSend(sleeper, false); SystemTestStateStoreFakeCommits commitsDsl = sleeper.stateStore().fakeCommits(); commitsDsl.send(factory -> factory.addPartitionFile("root", "file.parquet", 100)); - committer.addRunClearQueue(new StateStoreCommitterRun(Instant.now(), null, - List.of(new StateStoreCommitSummary(sleeper.tableProperties().get(TABLE_ID), "test-file-added", Instant.now())))); + committer.addRunClearQueue(new StateStoreCommitterRun("test-stream", Instant.now(), null, + List.of(new StateStoreCommitSummary("test-stream", sleeper.tableProperties().get(TABLE_ID), "test-file-added", Instant.now())))); // When / Then assertThatCode(() -> commitsDsl.waitForCommits(PollWithRetries.noRetries())) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java similarity index 64% rename from java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java rename to java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java index 9acb008bf1..59ebfd43c6 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunWaitForCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java @@ -21,12 +21,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.IntStream; -import static java.util.stream.Collectors.toUnmodifiableList; import static org.assertj.core.api.Assertions.assertThat; -public class StateStoreCommitterRunWaitForCommitsTest { +public class WaitForStateStoreCommitsTest { Map waitForNumCommitsByTableId = new HashMap<>(); @Test @@ -35,8 +33,8 @@ void shouldFindOneCommitWasMadeAgainstCorrectTable() { waitForNumCommitsByTableId.put("test-table", 2); // When - StateStoreCommitterRun.decrementWaitForNumCommits( - List.of(runWithCommitsToTable(1, "test-table")), + WaitForStateStoreCommits.decrementWaitForNumCommits( + List.of(commitToTable("test-table")), waitForNumCommitsByTableId); // Then @@ -49,8 +47,8 @@ void shouldFindAllCommitsWereMadeAgainstCorrectTable() { waitForNumCommitsByTableId.put("test-table", 2); // When - StateStoreCommitterRun.decrementWaitForNumCommits( - List.of(runWithCommitsToTable(2, "test-table")), + WaitForStateStoreCommits.decrementWaitForNumCommits( + List.of(commitToTable("test-table"), commitToTable("test-table")), waitForNumCommitsByTableId); // Then @@ -63,8 +61,8 @@ void shouldFindCommitAgainstWrongTable() { waitForNumCommitsByTableId.put("test-table", 2); // When - StateStoreCommitterRun.decrementWaitForNumCommits( - List.of(runWithCommitsToTable(1, "other-table")), + WaitForStateStoreCommits.decrementWaitForNumCommits( + List.of(commitToTable("other-table")), waitForNumCommitsByTableId); // Then @@ -78,8 +76,8 @@ void shouldFindCommitsAgainstMultipleTablesInOneRun() { waitForNumCommitsByTableId.put("table-2", 2); // When - StateStoreCommitterRun.decrementWaitForNumCommits( - List.of(runWithCommits(List.of(commitToTable("table-1"), commitToTable("table-2")))), + WaitForStateStoreCommits.decrementWaitForNumCommits( + List.of(commitToTable("table-1"), commitToTable("table-2")), waitForNumCommitsByTableId); // Then @@ -93,27 +91,17 @@ void shouldFindCommitsAgainstMultipleTablesInSeparateRuns() { waitForNumCommitsByTableId.put("table-2", 2); // When - StateStoreCommitterRun.decrementWaitForNumCommits(List.of( - runWithCommitsToTable(1, "table-1"), - runWithCommitsToTable(1, "table-2")), + WaitForStateStoreCommits.decrementWaitForNumCommits(List.of( + commitToTable("table-1"), + commitToTable("table-2")), waitForNumCommitsByTableId); // Then assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("table-1", 1, "table-2", 1)); } - private StateStoreCommitterRun runWithCommitsToTable(int commits, String tableId) { - return runWithCommits(IntStream.range(0, commits) - .mapToObj(i -> new StateStoreCommitSummary(tableId, "test-commit-type", Instant.now())) - .collect(toUnmodifiableList())); - } - - private StateStoreCommitterRun runWithCommits(List commits) { - return new StateStoreCommitterRun(Instant.now(), Instant.now(), commits); - } - private StateStoreCommitSummary commitToTable(String tableId) { - return new StateStoreCommitSummary(tableId, "test-commit-type", Instant.now()); + return new StateStoreCommitSummary("test-stream", tableId, "test-commit-type", Instant.now()); } } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index 7fa391688f..ccb99c1858 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -101,13 +101,13 @@ private void runCommitter() { try { StateStoreCommitRequest appliedRequest = committer.applyFromJson(message.getBody()); commits.add(new StateStoreCommitSummary( - appliedRequest.getTableId(), appliedRequest.getRequest().getClass().getSimpleName(), Instant.now())); + "test-stream", appliedRequest.getTableId(), appliedRequest.getRequest().getClass().getSimpleName(), Instant.now())); } catch (StateStoreException e) { throw new RuntimeException(e); } } if (!commits.isEmpty()) { - runs.add(new StateStoreCommitterRun(startTime, Instant.now(), commits)); + runs.add(new StateStoreCommitterRun("test-stream", startTime, Instant.now(), commits)); } } From ded3048b559a0ca95d7e9231396083feb9481711 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:38:21 +0000 Subject: [PATCH 30/49] Remove StateStoreCommitterRun from in-memory implementation --- .../statestore/StateStoreCommitterDriver.java | 4 ++- .../SystemTestStateStoreFakeCommitsTest.java | 6 +--- .../drivers/InMemoryStateStoreCommitter.java | 29 ++++++++++++------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java index 801f5976c1..bfade72fa5 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java @@ -25,7 +25,9 @@ public interface StateStoreCommitterDriver { void sendCommitMessages(Stream messages); - List getRunsInPeriod(Instant startTime, Instant endTime); + default List getRunsInPeriod(Instant startTime, Instant endTime) { + return List.of(); + } default List getLogsInPeriod(Instant startTime, Instant endTime) { return getRunsInPeriod(startTime, endTime) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index 11e3433ea3..6543f0a6fc 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -24,13 +24,10 @@ import sleeper.systemtest.dsl.testutil.InMemorySystemTestDrivers; import sleeper.systemtest.dsl.testutil.drivers.InMemoryStateStoreCommitter; -import java.time.Instant; -import java.util.List; import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; -import static sleeper.configuration.properties.table.TableProperty.TABLE_ID; import static sleeper.core.testutils.printers.FileReferencePrinter.printFiles; import static sleeper.systemtest.dsl.testutil.InMemoryTestInstance.MAIN; @@ -78,8 +75,7 @@ void shouldWaitForCommitWhenCommitWasMadeButRunIsUnfinished(SleeperSystemTest sl committer.setRunCommitterOnSend(sleeper, false); SystemTestStateStoreFakeCommits commitsDsl = sleeper.stateStore().fakeCommits(); commitsDsl.send(factory -> factory.addPartitionFile("root", "file.parquet", 100)); - committer.addRunClearQueue(new StateStoreCommitterRun("test-stream", Instant.now(), null, - List.of(new StateStoreCommitSummary("test-stream", sleeper.tableProperties().get(TABLE_ID), "test-file-added", Instant.now())))); + committer.fakeRunWithCommits(sleeper, 1); // When / Then assertThatCode(() -> commitsDsl.waitForCommits(PollWithRetries.noRetries())) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index ccb99c1858..026614b852 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -19,13 +19,16 @@ import sleeper.commit.StateStoreCommitter; import sleeper.configuration.properties.table.TablePropertiesProvider; import sleeper.core.statestore.StateStoreException; +import sleeper.core.table.TableNotFoundException; import sleeper.systemtest.dsl.SleeperSystemTest; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; -import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogEntry; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunFinished; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunStarted; import java.time.Instant; import java.util.ArrayList; @@ -44,7 +47,7 @@ public class InMemoryStateStoreCommitter { private final InMemoryIngestByQueue ingest; private final InMemoryCompaction compaction; private final Queue queue = new LinkedList<>(); - private final List runs = new ArrayList<>(); + private final List logs = new ArrayList<>(); private final Map runCommitterOnSendByTableId = new HashMap<>(); public InMemoryStateStoreCommitter(InMemoryIngestByQueue ingest, InMemoryCompaction compaction) { @@ -60,9 +63,11 @@ public void setRunCommitterOnSend(SleeperSystemTest sleeper, boolean runCommitte runCommitterOnSendByTableId.put(sleeper.tableProperties().get(TABLE_ID), runCommitterOnSend); } - public void addRunClearQueue(StateStoreCommitterRun run) { - runs.add(run); - queue.clear(); + public void fakeRunWithCommits(SleeperSystemTest sleeper, int numCommits) { + String tableId = sleeper.tableProperties().get(TABLE_ID); + logs.add(new StateStoreCommitterRunStarted("test-stream", Instant.now())); + logs.add(new StateStoreCommitterRunFinished("test-stream", Instant.now())); + logs.add(new StateStoreCommitSummary("test-stream", tableId, "test-commit", Instant.now())); } public class Driver implements StateStoreCommitterDriver { @@ -87,10 +92,10 @@ public void sendCommitMessages(Stream messages) { } @Override - public List getRunsInPeriod(Instant startTime, Instant endTime) { - return runs.stream() - .filter(run -> run.getStartTime().compareTo(startTime) >= 0) - .filter(run -> run.getStartTime().compareTo(endTime) <= 0) + public List getLogsInPeriod(Instant startTime, Instant endTime) { + return logs.stream() + .filter(run -> run.getTimeInCommitter().compareTo(startTime) >= 0) + .filter(run -> run.getTimeInCommitter().compareTo(endTime) <= 0) .collect(toUnmodifiableList()); } @@ -104,10 +109,14 @@ private void runCommitter() { "test-stream", appliedRequest.getTableId(), appliedRequest.getRequest().getClass().getSimpleName(), Instant.now())); } catch (StateStoreException e) { throw new RuntimeException(e); + } catch (TableNotFoundException e) { + // Discard messages from other tests } } if (!commits.isEmpty()) { - runs.add(new StateStoreCommitterRun("test-stream", startTime, Instant.now(), commits)); + logs.add(new StateStoreCommitterRunStarted("test-stream", startTime)); + logs.addAll(commits); + logs.add(new StateStoreCommitterRunFinished("test-stream", Instant.now())); } } From 5ee106d2913c2ac5230041c10bd0e4cd00eb218e Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:41:29 +0000 Subject: [PATCH 31/49] Rename ReadStateStoreCommitterLogs --- ....java => ReadStateStoreCommitterLogs.java} | 27 +++++++++++++++++-- .../StateStoreCommitterRunsBuilder.java | 6 ++--- ...a => ReadStateStoreCommitterLogsTest.java} | 14 +++++----- 3 files changed, 35 insertions(+), 12 deletions(-) rename java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/{StateStoreCommitterLogEntry.java => ReadStateStoreCommitterLogs.java} (83%) rename java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/{StateStoreCommitterLogEntryTest.java => ReadStateStoreCommitterLogsTest.java} (81%) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java similarity index 83% rename from java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java rename to java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java index 407a15e08c..cb5fdd42bd 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java @@ -15,14 +15,17 @@ */ package sleeper.systemtest.drivers.statestore; +import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; + import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class StateStoreCommitterLogEntry { +public class ReadStateStoreCommitterLogs { private static final Pattern MESSAGE_PATTERN = Pattern.compile("" + "Lambda started at (.+)|" + // Lambda started message type @@ -40,7 +43,7 @@ private CapturingGroups() { } } - private StateStoreCommitterLogEntry() { + private ReadStateStoreCommitterLogs() { } public static Object readEvent(String logStream, String message) { @@ -68,6 +71,26 @@ public static Object readEvent(String logStream, String message) { return null; } + public static Object readEvent(List entry) { + String logStream = null; + String message = null; + for (ResultField field : entry) { + switch (field.field()) { + case "@logStream": + logStream = field.value(); + break; + case "@message": + message = field.value(); + break; + default: + break; + } + } + return readEvent( + Objects.requireNonNull(logStream, "Log stream not found"), + Objects.requireNonNull(message, "Log message not found")); + } + public static class LambdaStarted { private final Instant startTime; diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java index 425ed5264a..68197d9bb6 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java @@ -17,8 +17,8 @@ import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; -import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaFinished; -import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaStarted; +import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaFinished; +import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaStarted; import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; @@ -54,7 +54,7 @@ public void add(List entry) { Objects.requireNonNull(logStream, "Log stream not found"); Objects.requireNonNull(message, "Log message not found"); logStreamByName.computeIfAbsent(logStream, name -> new LogStream()) - .add(StateStoreCommitterLogEntry.readEvent(logStream, message)); + .add(ReadStateStoreCommitterLogs.readEvent(logStream, message)); } public List buildRuns() { diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java similarity index 81% rename from java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java rename to java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java index afa6fc567a..e649a71d33 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterLogEntryTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java @@ -17,15 +17,15 @@ import org.junit.jupiter.api.Test; -import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaFinished; -import sleeper.systemtest.drivers.statestore.StateStoreCommitterLogEntry.LambdaStarted; +import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaFinished; +import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaStarted; import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import java.time.Instant; import static org.assertj.core.api.Assertions.assertThat; -public class StateStoreCommitterLogEntryTest { +public class ReadStateStoreCommitterLogsTest { @Test void shouldReadLambdaStarted() { @@ -33,7 +33,7 @@ void shouldReadLambdaStarted() { String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isEqualTo( + assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isEqualTo( new LambdaStarted(Instant.parse("2024-08-13T12:12:00Z"))); } @@ -43,7 +43,7 @@ void shouldReadLambdaFinished() { String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isEqualTo( + assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isEqualTo( new LambdaFinished(Instant.parse("2024-08-13T12:13:00Z"))); } @@ -53,7 +53,7 @@ void shouldReadCommitApplied() { String message = "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isEqualTo( + assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isEqualTo( new StateStoreCommitSummary("test-stream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))); } @@ -63,6 +63,6 @@ void shouldReadUnrecognisedLog() { String message = "some other log"; // When / Then - assertThat(StateStoreCommitterLogEntry.readEvent("test-stream", message)).isNull(); + assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isNull(); } } From 1bdb0bf52180eab67496c2e89506c0d2aa8948e7 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:52:05 +0000 Subject: [PATCH 32/49] Remove StateStoreCommitterRun --- .../AwsStateStoreCommitterDriver.java | 11 +- .../ReadStateStoreCommitterLogs.java | 13 +- .../StateStoreCommitterRunsBuilder.java | 122 ------------- .../ReadStateStoreCommitterLogsTest.java | 56 +++++- .../StateStoreCommitterRunsBuilderTest.java | 161 ------------------ .../statestore/StateStoreCommitterDriver.java | 12 +- .../statestore/StateStoreCommitterRun.java | 79 --------- .../StateStoreCommitterRunsBuilder.java | 20 --- 8 files changed, 62 insertions(+), 412 deletions(-) delete mode 100644 java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java delete mode 100644 java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java delete mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java delete mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java index 32132374ff..963cda1c87 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java @@ -29,7 +29,7 @@ import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; -import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogEntry; import java.time.Duration; import java.time.Instant; @@ -61,7 +61,7 @@ public void sendCommitMessages(Stream messages) { } @Override - public List getRunsInPeriod(Instant startTime, Instant endTime) { + public List getLogsInPeriod(Instant startTime, Instant endTime) { String logGroupName = instance.getInstanceProperties().get(STATESTORE_COMMITTER_LOG_GROUP); LOGGER.info("Submitting logs query for log group {} starting at time {}", logGroupName, startTime); String queryId = cloudWatch.startQuery(builder -> builder @@ -73,10 +73,9 @@ public List getRunsInPeriod(Instant startTime, Instant e "| filter @message like /Lambda (started|finished) at|Applied request to table/ " + "| sort @timestamp asc")) .queryId(); - GetQueryResultsResponse response = waitForQuery(queryId); - StateStoreCommitterRunsBuilder builder = new StateStoreCommitterRunsBuilder(); - response.results().forEach(builder::add); - return builder.buildRuns(); + return waitForQuery(queryId).results().stream() + .map(ReadStateStoreCommitterLogs::read) + .collect(toUnmodifiableList()); } private void sendMessageBatch(List batch) { diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java index cb5fdd42bd..6f97533683 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java @@ -18,6 +18,9 @@ import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogEntry; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunFinished; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunStarted; import java.time.Instant; import java.util.List; @@ -46,7 +49,7 @@ private CapturingGroups() { private ReadStateStoreCommitterLogs() { } - public static Object readEvent(String logStream, String message) { + private static StateStoreCommitterLogEntry readMessage(String logStream, String message) { Matcher matcher = MESSAGE_PATTERN.matcher(message); if (!matcher.find()) { return null; @@ -56,11 +59,11 @@ public static Object readEvent(String logStream, String message) { // We determine which type of message was found based on which capturing group is set. String startTime = matcher.group(CapturingGroups.START_TIME); if (startTime != null) { - return new LambdaStarted(Instant.parse(startTime)); + return new StateStoreCommitterRunStarted(logStream, Instant.parse(startTime)); } String finishTime = matcher.group(CapturingGroups.FINISH_TIME); if (finishTime != null) { - return new LambdaFinished(Instant.parse(finishTime)); + return new StateStoreCommitterRunFinished(logStream, Instant.parse(finishTime)); } String tableId = matcher.group(CapturingGroups.TABLE_ID); if (tableId != null) { @@ -71,7 +74,7 @@ public static Object readEvent(String logStream, String message) { return null; } - public static Object readEvent(List entry) { + public static StateStoreCommitterLogEntry read(List entry) { String logStream = null; String message = null; for (ResultField field : entry) { @@ -86,7 +89,7 @@ public static Object readEvent(List entry) { break; } } - return readEvent( + return readMessage( Objects.requireNonNull(logStream, "Log stream not found"), Objects.requireNonNull(message, "Log message not found")); } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java deleted file mode 100644 index 68197d9bb6..0000000000 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilder.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2022-2024 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.systemtest.drivers.statestore; - -import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; - -import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaFinished; -import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaStarted; -import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; -import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toUnmodifiableList; - -public class StateStoreCommitterRunsBuilder { - - private final Map logStreamByName = new LinkedHashMap<>(); - - public void add(List entry) { - String logStream = null; - String message = null; - for (ResultField field : entry) { - switch (field.field()) { - case "@logStream": - logStream = field.value(); - break; - case "@message": - message = field.value(); - break; - default: - break; - } - } - Objects.requireNonNull(logStream, "Log stream not found"); - Objects.requireNonNull(message, "Log message not found"); - logStreamByName.computeIfAbsent(logStream, name -> new LogStream()) - .add(ReadStateStoreCommitterLogs.readEvent(logStream, message)); - } - - public List buildRuns() { - return logStreamByName.entrySet().stream() - .flatMap(entry -> entry.getValue().runs() - .map(run -> run.build(entry.getKey()))) - .collect(toUnmodifiableList()); - } - - private static class LogStream { - - private final List runs = new ArrayList<>(); - private LambdaRun lastRun; - - void add(Object event) { - if (event instanceof LambdaStarted) { - lastRun = new LambdaRun((LambdaStarted) event); - runs.add(lastRun); - } else if (event instanceof LambdaFinished) { - lastRunOrUnknownStartTime().finished((LambdaFinished) event); - } else if (event instanceof StateStoreCommitSummary) { - lastRunOrUnknownStartTime().committed((StateStoreCommitSummary) event); - } - } - - private LambdaRun lastRunOrUnknownStartTime() { - if (lastRun == null) { - lastRun = new LambdaRun(); - runs.add(lastRun); - } - return lastRun; - } - - Stream runs() { - return runs.stream(); - } - } - - private static class LambdaRun { - private final Instant startTime; - private Instant finishTime; - private List commits = new ArrayList<>(); - - LambdaRun() { - startTime = null; - } - - LambdaRun(LambdaStarted event) { - this.startTime = event.getStartTime(); - } - - void finished(LambdaFinished event) { - finishTime = event.getFinishTime(); - } - - void committed(StateStoreCommitSummary commit) { - commits.add(commit); - } - - StateStoreCommitterRun build(String logStream) { - return new StateStoreCommitterRun(logStream, startTime, finishTime, commits); - } - } - -} diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java index e649a71d33..75d3314f11 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java @@ -16,12 +16,14 @@ package sleeper.systemtest.drivers.statestore; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; -import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaFinished; -import sleeper.systemtest.drivers.statestore.ReadStateStoreCommitterLogs.LambdaStarted; import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunFinished; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunStarted; import java.time.Instant; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -33,8 +35,8 @@ void shouldReadLambdaStarted() { String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; // When / Then - assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isEqualTo( - new LambdaStarted(Instant.parse("2024-08-13T12:12:00Z"))); + assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isEqualTo( + new StateStoreCommitterRunStarted("test-stream", Instant.parse("2024-08-13T12:12:00Z"))); } @Test @@ -43,8 +45,8 @@ void shouldReadLambdaFinished() { String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"; // When / Then - assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isEqualTo( - new LambdaFinished(Instant.parse("2024-08-13T12:13:00Z"))); + assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isEqualTo( + new StateStoreCommitterRunFinished("test-stream", Instant.parse("2024-08-13T12:13:00Z"))); } @Test @@ -53,7 +55,7 @@ void shouldReadCommitApplied() { String message = "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"; // When / Then - assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isEqualTo( + assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isEqualTo( new StateStoreCommitSummary("test-stream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))); } @@ -63,6 +65,44 @@ void shouldReadUnrecognisedLog() { String message = "some other log"; // When / Then - assertThat(ReadStateStoreCommitterLogs.readEvent("test-stream", message)).isNull(); + assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isNull(); + } + + @Test + void shouldReadLogWithFieldsInReverseOrder() { + // Given + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + + // When / Then + assertThat(ReadStateStoreCommitterLogs.read(logEntryReversed("test-stream", message))).isEqualTo( + new StateStoreCommitterRunStarted("test-stream", Instant.parse("2024-08-13T12:12:00Z"))); + } + + @Test + void shouldReadLogWithExtraFields() { + // Given + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + List entry = List.of( + ResultField.builder().field("@extraField").value("some value").build(), + ResultField.builder().field("@message").value(message).build(), + ResultField.builder().field("@otherField").value("other value").build(), + ResultField.builder().field("@logStream").value("test-stream").build(), + ResultField.builder().field("@anotherField").value("another value").build()); + + // When / Then + assertThat(ReadStateStoreCommitterLogs.read(entry)) + .isEqualTo(new StateStoreCommitterRunStarted("test-stream", Instant.parse("2024-08-13T12:12:00Z"))); + } + + private List logEntry(String logStream, String message) { + return List.of( + ResultField.builder().field("@logStream").value(logStream).build(), + ResultField.builder().field("@message").value(message).build()); + } + + private List logEntryReversed(String logStream, String message) { + return List.of( + ResultField.builder().field("@message").value(message).build(), + ResultField.builder().field("@logStream").value(logStream).build()); } } diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java deleted file mode 100644 index 8abc7158f5..0000000000 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/StateStoreCommitterRunsBuilderTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2022-2024 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.systemtest.drivers.statestore; - -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.services.cloudwatchlogs.model.ResultField; - -import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; -import sleeper.systemtest.dsl.statestore.StateStoreCommitterRun; - -import java.time.Instant; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class StateStoreCommitterRunsBuilderTest { - StateStoreCommitterRunsBuilder builder = new StateStoreCommitterRunsBuilder(); - - @Test - void shouldBuildSingleRunNoCommits() { - // Given - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); - } - - @Test - void shouldBuildOverlappingRunsOnTwoLogStreams() { - // Given - add("stream-1", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); - add("stream-2", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:30Z"); - add("stream-1", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); - add("stream-2", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:30Z (ran for 1 minute)"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("stream-1", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of()), - new StateStoreCommitterRun("stream-2", Instant.parse("2024-08-13T12:12:30Z"), Instant.parse("2024-08-13T12:13:30Z"), List.of())); - } - - @Test - void shouldBuildSingleRunWithOneCommit() { - // Given - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); - add("test-logstream", "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"); - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), - List.of(new StateStoreCommitSummary("test-logstream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); - } - - @Test - void shouldBuildUnfinishedRun() { - // Given - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-stream", Instant.parse("2024-08-13T12:12:00Z"), null, List.of())); - } - - @Test - void shouldBuildUnfinishedRunWithOneCommit() { - // Given - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); - add("test-logstream", "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), null, - List.of(new StateStoreCommitSummary("test-logstream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); - } - - @Test - void shouldBuildRunWithOneCommitAndNoStartMessage() { - // Given - add("test-logstream", "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", null, null, - List.of(new StateStoreCommitSummary("test-logstream", "test-table", "TestRequest", Instant.parse("2024-08-13T12:12:30Z"))))); - } - - @Test - void shouldBuildRunWithOnlyFinishMessage() { - // Given - add("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:30Z (ran for 1 minute)"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", null, Instant.parse("2024-08-13T12:13:30Z"), List.of())); - } - - @Test - void shouldIgnoreUnrecognisedLog() { - // Given - add("test-logstream", "Some unknown message"); - - // When / Then - assertThat(builder.buildRuns()).isEmpty(); - } - - @Test - void shouldReadLogWithFieldsInReverseOrder() { - // Given - addReversingFields("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"); - addReversingFields("test-logstream", "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); - } - - @Test - void shouldIgnoreExtraFieldsInLogMessage() { - // Given - builder.add(List.of( - ResultField.builder().field("@logStream").value("test-logstream").build(), - ResultField.builder().field("@someField").value("Some ignored field").build(), - ResultField.builder().field("@message").value("[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z").build())); - builder.add(List.of( - ResultField.builder().field("@logStream").value("test-logstream").build(), - ResultField.builder().field("@message").value("[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)").build(), - ResultField.builder().field("@otherField").value("Other ignored field").build())); - - // When / Then - assertThat(builder.buildRuns()).containsExactly( - new StateStoreCommitterRun("test-logstream", Instant.parse("2024-08-13T12:12:00Z"), Instant.parse("2024-08-13T12:13:00Z"), List.of())); - } - - void add(String logStream, String message) { - builder.add(List.of( - ResultField.builder().field("@logStream").value(logStream).build(), - ResultField.builder().field("@message").value(message).build())); - } - - void addReversingFields(String logStream, String message) { - builder.add(List.of( - ResultField.builder().field("@message").value(message).build(), - ResultField.builder().field("@logStream").value(logStream).build())); - } -} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java index bfade72fa5..f9a1b3ac54 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java @@ -19,19 +19,9 @@ import java.util.List; import java.util.stream.Stream; -import static java.util.stream.Collectors.toUnmodifiableList; - public interface StateStoreCommitterDriver { void sendCommitMessages(Stream messages); - default List getRunsInPeriod(Instant startTime, Instant endTime) { - return List.of(); - } - - default List getLogsInPeriod(Instant startTime, Instant endTime) { - return getRunsInPeriod(startTime, endTime) - .stream().flatMap(run -> run.logs()) - .collect(toUnmodifiableList()); - } + List getLogsInPeriod(Instant startTime, Instant endTime); } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java deleted file mode 100644 index 6b31dd61ca..0000000000 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRun.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2022-2024 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.systemtest.dsl.statestore; - -import java.time.Instant; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Stream; - -public class StateStoreCommitterRun { - - private final String logStream; - private final Instant startTime; - private final Instant finishTime; - private final List commits; - - public StateStoreCommitterRun(String logStream, Instant startTime, Instant finishTime, List commits) { - this.logStream = logStream; - this.startTime = startTime; - this.finishTime = finishTime; - this.commits = commits; - } - - public Instant getStartTime() { - return startTime; - } - - public Instant getFinishTime() { - return finishTime; - } - - public List getCommits() { - return commits; - } - - public Stream logs() { - return Stream.of( - Optional.ofNullable(startTime).map(time -> new StateStoreCommitterRunStarted(logStream, time)).stream(), - commits.stream(), - Optional.ofNullable(finishTime).map(time -> new StateStoreCommitterRunFinished(logStream, time)).stream()) - .flatMap(s -> s); - } - - @Override - public int hashCode() { - return Objects.hash(startTime, finishTime, commits); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof StateStoreCommitterRun)) { - return false; - } - StateStoreCommitterRun other = (StateStoreCommitterRun) obj; - return Objects.equals(startTime, other.startTime) && Objects.equals(finishTime, other.finishTime) && Objects.equals(commits, other.commits); - } - - @Override - public String toString() { - return "StateStoreCommitterRun{startTime=" + startTime + ", finishTime=" + finishTime + ", commits=" + commits + "}"; - } -} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java deleted file mode 100644 index 649c46835c..0000000000 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterRunsBuilder.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2022-2024 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.systemtest.dsl.statestore; - -public class StateStoreCommitterRunsBuilder { - -} From 8f61246222bebb494c4dd5de069bdcb272d1c58a Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:53:24 +0000 Subject: [PATCH 33/49] Use StateStoreCommitterLogEntry.getLastTime --- .../dsl/statestore/SystemTestStateStoreFakeCommits.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index c8d85a06cc..096f77458d 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -24,7 +24,6 @@ import java.time.Duration; import java.time.Instant; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -62,9 +61,7 @@ public SystemTestStateStoreFakeCommits waitForCommits(PollWithRetries poll) thro poll.pollUntil("all state store commits are applied", () -> { List logs = driver.getLogsInPeriod(getRunsAfterTime, Instant.now().plus(QUERY_RUNS_TIME_SLACK)); WaitForStateStoreCommits.decrementWaitForNumCommits(logs, waitForNumCommitsByTableId); - logs.stream() - .map(StateStoreCommitterLogEntry::getTimeInCommitter) - .max(Comparator.naturalOrder()) + StateStoreCommitterLogEntry.getLastTime(logs) .ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_TIME_SLACK)); LOGGER.info("Remaining unapplied commits by table ID: {}", waitForNumCommitsByTableId); return waitForNumCommitsByTableId.isEmpty(); From d9f53d0125a90520933a158245a92a6e75409358 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:30:52 +0000 Subject: [PATCH 34/49] Test logs processing in WaitForStateStoreCommits --- .../AwsStateStoreCommitterDriver.java | 57 +------ .../AwsStateStoreCommitterLogsDriver.java | 91 ++++++++++ .../drivers/util/AwsSystemTestDrivers.java | 9 +- .../systemtest/dsl/SystemTestDrivers.java | 3 + .../statestore/StateStoreCommitterDriver.java | 4 - .../StateStoreCommitterLogsDriver.java | 24 +++ .../SystemTestStateStoreFakeCommits.java | 25 +-- .../statestore/WaitForStateStoreCommits.java | 33 +++- ...ateStoreCommitterLogEntryLastTimeTest.java | 159 ------------------ .../WaitForStateStoreCommitsTest.java | 79 +++++---- .../testutil/InMemorySystemTestDrivers.java | 6 + .../drivers/InMemoryStateStoreCommitter.java | 20 ++- 12 files changed, 224 insertions(+), 286 deletions(-) create mode 100644 java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterLogsDriver.java create mode 100644 java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogsDriver.java delete mode 100644 java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java index 963cda1c87..8642dd6b83 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java @@ -20,26 +20,17 @@ import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; -import software.amazon.awssdk.services.cloudwatchlogs.model.GetQueryResultsResponse; -import software.amazon.awssdk.services.cloudwatchlogs.model.QueryStatus; -import sleeper.core.util.PollWithRetries; import sleeper.core.util.SplitIntoBatches; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; -import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogEntry; -import java.time.Duration; -import java.time.Instant; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.stream.Stream; import static java.util.stream.Collectors.toUnmodifiableList; -import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_LOG_GROUP; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_QUEUE_URL; public class AwsStateStoreCommitterDriver implements StateStoreCommitterDriver { @@ -47,12 +38,10 @@ public class AwsStateStoreCommitterDriver implements StateStoreCommitterDriver { private final SystemTestInstanceContext instance; private final AmazonSQS sqs; - private final CloudWatchLogsClient cloudWatch; - public AwsStateStoreCommitterDriver(SystemTestInstanceContext instance, AmazonSQS sqs, CloudWatchLogsClient cloudWatch) { + public AwsStateStoreCommitterDriver(SystemTestInstanceContext instance, AmazonSQS sqs) { this.instance = instance; this.sqs = sqs; - this.cloudWatch = cloudWatch; } @Override @@ -60,24 +49,6 @@ public void sendCommitMessages(Stream messages) { SplitIntoBatches.inParallelBatchesOf(10, messages, this::sendMessageBatch); } - @Override - public List getLogsInPeriod(Instant startTime, Instant endTime) { - String logGroupName = instance.getInstanceProperties().get(STATESTORE_COMMITTER_LOG_GROUP); - LOGGER.info("Submitting logs query for log group {} starting at time {}", logGroupName, startTime); - String queryId = cloudWatch.startQuery(builder -> builder - .logGroupName(logGroupName) - .startTime(startTime.getEpochSecond()) - .endTime(endTime.getEpochSecond()) - .limit(10000) - .queryString("fields @timestamp, @message, @logStream " + - "| filter @message like /Lambda (started|finished) at|Applied request to table/ " + - "| sort @timestamp asc")) - .queryId(); - return waitForQuery(queryId).results().stream() - .map(ReadStateStoreCommitterLogs::read) - .collect(toUnmodifiableList()); - } - private void sendMessageBatch(List batch) { sqs.sendMessageBatch(new SendMessageBatchRequest() .withQueueUrl(instance.getInstanceProperties().get(STATESTORE_COMMITTER_QUEUE_URL)) @@ -89,30 +60,4 @@ private void sendMessageBatch(List batch) { .withMessageBody(message.getBody())) .collect(toUnmodifiableList()))); } - - private GetQueryResultsResponse waitForQuery(String queryId) { - try { - return PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(1), Duration.ofMinutes(1)) - .queryUntil("query is completed", - () -> cloudWatch.getQueryResults(builder -> builder.queryId(queryId)), - results -> isQueryCompleted(results)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } - - private static boolean isQueryCompleted(GetQueryResultsResponse response) { - LOGGER.info("Logs query response status {}, statistics: {}", - response.statusAsString(), response.statistics()); - QueryStatus status = response.status(); - if (status == QueryStatus.COMPLETE) { - return true; - } else if (Set.of(QueryStatus.SCHEDULED, QueryStatus.RUNNING).contains(status)) { - return false; - } else { - throw new RuntimeException("Logs query failed with status " + response.statusAsString()); - } - } - } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterLogsDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterLogsDriver.java new file mode 100644 index 0000000000..9f1442b545 --- /dev/null +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterLogsDriver.java @@ -0,0 +1,91 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.drivers.statestore; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; +import software.amazon.awssdk.services.cloudwatchlogs.model.GetQueryResultsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.QueryStatus; + +import sleeper.core.util.PollWithRetries; +import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogEntry; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogsDriver; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static java.util.stream.Collectors.toUnmodifiableList; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_LOG_GROUP; + +public class AwsStateStoreCommitterLogsDriver implements StateStoreCommitterLogsDriver { + public static final Logger LOGGER = LoggerFactory.getLogger(AwsStateStoreCommitterDriver.class); + + private final SystemTestInstanceContext instance; + private final CloudWatchLogsClient cloudWatch; + + public AwsStateStoreCommitterLogsDriver(SystemTestInstanceContext instance, CloudWatchLogsClient cloudWatch) { + this.instance = instance; + this.cloudWatch = cloudWatch; + } + + @Override + public List getLogsInPeriod(Instant startTime, Instant endTime) { + String logGroupName = instance.getInstanceProperties().get(STATESTORE_COMMITTER_LOG_GROUP); + LOGGER.info("Submitting logs query for log group {} starting at time {}", logGroupName, startTime); + String queryId = cloudWatch.startQuery(builder -> builder + .logGroupName(logGroupName) + .startTime(startTime.getEpochSecond()) + .endTime(endTime.getEpochSecond()) + .limit(10000) + .queryString("fields @timestamp, @message, @logStream " + + "| filter @message like /Lambda (started|finished) at|Applied request to table/ " + + "| sort @timestamp asc")) + .queryId(); + return waitForQuery(queryId).results().stream() + .map(ReadStateStoreCommitterLogs::read) + .collect(toUnmodifiableList()); + } + + private GetQueryResultsResponse waitForQuery(String queryId) { + try { + return PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(1), Duration.ofMinutes(1)) + .queryUntil("query is completed", + () -> cloudWatch.getQueryResults(builder -> builder.queryId(queryId)), + results -> isQueryCompleted(results)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static boolean isQueryCompleted(GetQueryResultsResponse response) { + LOGGER.info("Logs query response status {}, statistics: {}", + response.statusAsString(), response.statistics()); + QueryStatus status = response.status(); + if (status == QueryStatus.COMPLETE) { + return true; + } else if (Set.of(QueryStatus.SCHEDULED, QueryStatus.RUNNING).contains(status)) { + return false; + } else { + throw new RuntimeException("Logs query failed with status " + response.statusAsString()); + } + } + +} diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java index bb24ff4c8c..947eb17fa0 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/util/AwsSystemTestDrivers.java @@ -46,6 +46,7 @@ import sleeper.systemtest.drivers.sourcedata.AwsIngestSourceFilesDriver; import sleeper.systemtest.drivers.statestore.AwsSnapshotsDriver; import sleeper.systemtest.drivers.statestore.AwsStateStoreCommitterDriver; +import sleeper.systemtest.drivers.statestore.AwsStateStoreCommitterLogsDriver; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.SystemTestDrivers; import sleeper.systemtest.dsl.compaction.CompactionDriver; @@ -76,6 +77,7 @@ import sleeper.systemtest.dsl.sourcedata.GeneratedIngestSourceFilesDriver; import sleeper.systemtest.dsl.sourcedata.IngestSourceFilesDriver; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogsDriver; import sleeper.systemtest.dsl.util.PurgeQueueDriver; import sleeper.systemtest.dsl.util.WaitForJobs; @@ -112,7 +114,12 @@ public SleeperTablesDriver tables(SystemTestParameters parameters) { @Override public StateStoreCommitterDriver stateStoreCommitter(SystemTestContext context) { - return new AwsStateStoreCommitterDriver(context.instance(), clients.getSqs(), clients.getCloudWatchLogs()); + return new AwsStateStoreCommitterDriver(context.instance(), clients.getSqs()); + } + + @Override + public StateStoreCommitterLogsDriver stateStoreCommitterLogs(SystemTestContext context) { + return new AwsStateStoreCommitterLogsDriver(context.instance(), clients.getCloudWatchLogs()); } @Override diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/SystemTestDrivers.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/SystemTestDrivers.java index 9bd10f3759..e186b851a7 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/SystemTestDrivers.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/SystemTestDrivers.java @@ -44,6 +44,7 @@ import sleeper.systemtest.dsl.sourcedata.GeneratedIngestSourceFilesDriver; import sleeper.systemtest.dsl.sourcedata.IngestSourceFilesDriver; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogsDriver; import sleeper.systemtest.dsl.util.PurgeQueueDriver; import sleeper.systemtest.dsl.util.WaitForJobs; @@ -68,6 +69,8 @@ public interface SystemTestDrivers { StateStoreCommitterDriver stateStoreCommitter(SystemTestContext context); + StateStoreCommitterLogsDriver stateStoreCommitterLogs(SystemTestContext context); + PartitionSplittingDriver partitionSplitting(SystemTestContext context); DirectIngestDriver directIngest(SystemTestContext context); diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java index f9a1b3ac54..3ac9cb5696 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterDriver.java @@ -15,13 +15,9 @@ */ package sleeper.systemtest.dsl.statestore; -import java.time.Instant; -import java.util.List; import java.util.stream.Stream; public interface StateStoreCommitterDriver { void sendCommitMessages(Stream messages); - - List getLogsInPeriod(Instant startTime, Instant endTime); } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogsDriver.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogsDriver.java new file mode 100644 index 0000000000..8319a23299 --- /dev/null +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogsDriver.java @@ -0,0 +1,24 @@ +/* + * Copyright 2022-2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.systemtest.dsl.statestore; + +import java.time.Instant; +import java.util.List; + +public interface StateStoreCommitterLogsDriver { + + List getLogsInPeriod(Instant startTime, Instant endTime); +} diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index 096f77458d..53967048ae 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -20,11 +20,10 @@ import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SystemTestContext; +import sleeper.systemtest.dsl.SystemTestDrivers; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; -import java.time.Duration; import java.time.Instant; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -33,17 +32,18 @@ public class SystemTestStateStoreFakeCommits { public static final Logger LOGGER = LoggerFactory.getLogger(SystemTestStateStoreFakeCommits.class); - private static final Duration QUERY_RUNS_TIME_SLACK = Duration.ofSeconds(5); - private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; + private final WaitForStateStoreCommits waiter; private final Map waitForNumCommitsByTableId = new ConcurrentHashMap<>(); - private Instant getRunsAfterTime; + private final Instant getRunsAfterTime; public SystemTestStateStoreFakeCommits(SystemTestContext context) { instance = context.instance(); - driver = context.instance().adminDrivers().stateStoreCommitter(context); - getRunsAfterTime = context.reporting().getRecordingStartTime().minus(QUERY_RUNS_TIME_SLACK); + SystemTestDrivers adminDrivers = context.instance().adminDrivers(); + driver = adminDrivers.stateStoreCommitter(context); + waiter = new WaitForStateStoreCommits(adminDrivers.stateStoreCommitterLogs(context)); + getRunsAfterTime = context.reporting().getRecordingStartTime(); } public SystemTestStateStoreFakeCommits sendBatched(Function> buildCommits) { @@ -57,15 +57,8 @@ public SystemTestStateStoreFakeCommits send(Function { - List logs = driver.getLogsInPeriod(getRunsAfterTime, Instant.now().plus(QUERY_RUNS_TIME_SLACK)); - WaitForStateStoreCommits.decrementWaitForNumCommits(logs, waitForNumCommitsByTableId); - StateStoreCommitterLogEntry.getLastTime(logs) - .ifPresent(lastTime -> getRunsAfterTime = lastTime.minus(QUERY_RUNS_TIME_SLACK)); - LOGGER.info("Remaining unapplied commits by table ID: {}", waitForNumCommitsByTableId); - return waitForNumCommitsByTableId.isEmpty(); - }); + waiter.waitForCommits(poll, waitForNumCommitsByTableId, getRunsAfterTime); + waitForNumCommitsByTableId.clear(); return this; } diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java index 2fb6ec9449..7e925bca1d 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java @@ -15,6 +15,14 @@ */ package sleeper.systemtest.dsl.statestore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sleeper.core.util.PollWithRetries; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,8 +30,31 @@ import static java.util.stream.Collectors.summingInt; public class WaitForStateStoreCommits { + public static final Logger LOGGER = LoggerFactory.getLogger(WaitForStateStoreCommits.class); + private static final Duration QUERY_RUNS_TIME_SLACK = Duration.ofSeconds(5); + + private final StateStoreCommitterLogsDriver driver; + + public WaitForStateStoreCommits(StateStoreCommitterLogsDriver driver) { + this.driver = driver; + } + + public void waitForCommits(PollWithRetries poll, Map waitForNumCommitsByTableId, Instant getRunsAfterTime) throws InterruptedException { + LOGGER.info("Waiting for commits by table ID: {}", waitForNumCommitsByTableId); + Instant startTime = getRunsAfterTime.minus(QUERY_RUNS_TIME_SLACK); + poll.pollUntil("all state store commits are applied", () -> { + Instant endTime = Instant.now().plus(QUERY_RUNS_TIME_SLACK); + return getRemainingCommitsInPeriod(waitForNumCommitsByTableId, startTime, endTime) + .isEmpty(); + }); + } - private WaitForStateStoreCommits() { + public Map getRemainingCommitsInPeriod(Map waitForNumCommitsByTableId, Instant startTime, Instant endTime) { + Map remainingCommitsByTableId = new HashMap<>(waitForNumCommitsByTableId); + List logs = driver.getLogsInPeriod(startTime, endTime); + WaitForStateStoreCommits.decrementWaitForNumCommits(logs, remainingCommitsByTableId); + LOGGER.info("Remaining unapplied commits by table ID: {}", remainingCommitsByTableId); + return remainingCommitsByTableId; } public static void decrementWaitForNumCommits(List entries, Map waitForNumCommitsByTableId) { diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java deleted file mode 100644 index f883d442bc..0000000000 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntryLastTimeTest.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright 2022-2024 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.systemtest.dsl.statestore; - -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class StateStoreCommitterLogEntryLastTimeTest { - - @Test - void shouldGetStartTimeWhenRunJustStarted() { - // Given - Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); - List entries = List.of(runStarted(startTime)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(startTime); - } - - @Test - void shouldGetFinishTimeWhenRunFinished() { - // Given - Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); - Instant finishTime = Instant.parse("2024-08-14T09:58:10Z"); - List entries = List.of(runStarted(startTime), runFinished(finishTime)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(finishTime); - } - - @Test - void shouldGetFinishTimeWhenRunFinishedWithACommit() { - // Given - Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); - Instant commitTime = Instant.parse("2024-08-14T09:58:05Z"); - Instant finishTime = Instant.parse("2024-08-14T09:58:10Z"); - List entries = List.of( - runStarted(startTime), commitAtTime(commitTime), runFinished(finishTime)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(finishTime); - } - - @Test - void shouldGetCommitTimeWhenRunUnfinishedWithACommit() { - // Given - Instant startTime = Instant.parse("2024-08-14T09:58:00Z"); - Instant commitTime = Instant.parse("2024-08-14T09:58:05Z"); - List entries = List.of(runStarted(startTime), commitAtTime(commitTime)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(commitTime); - } - - @Test - void shouldGetLatestRunWhenLastInList() { - // Given - Instant runTime1 = Instant.parse("2024-08-14T09:58:00Z"); - Instant runTime2 = Instant.parse("2024-08-14T09:59:00Z"); - List entries = List.of(runStarted(runTime1), runStarted(runTime2)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(runTime2); - } - - @Test - void shouldGetLatestRunWhenFirstInList() { - // Given - Instant runTime1 = Instant.parse("2024-08-14T09:58:00Z"); - Instant runTime2 = Instant.parse("2024-08-14T09:59:00Z"); - List entries = List.of(runStarted(runTime2), runStarted(runTime1)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(runTime2); - } - - @Test - void shouldGetLatestCommitWhenLastInList() { - // Given - Instant runTime = Instant.parse("2024-08-14T09:58:00Z"); - Instant commitTime1 = Instant.parse("2024-08-14T09:58:01Z"); - Instant commitTime2 = Instant.parse("2024-08-14T09:58:02Z"); - List entries = List.of( - runStarted(runTime), commitAtTime(commitTime1), commitAtTime(commitTime2)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(commitTime2); - } - - @Test - void shouldGetLatestCommitWhenFirstInList() { - // Given - Instant runTime = Instant.parse("2024-08-14T09:58:00Z"); - Instant commitTime1 = Instant.parse("2024-08-14T09:58:01Z"); - Instant commitTime2 = Instant.parse("2024-08-14T09:58:02Z"); - List entries = List.of( - runStarted(runTime), commitAtTime(commitTime2), commitAtTime(commitTime1)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(commitTime2); - } - - @Test - void shouldGetCommitTimeWhenRunStartTimeUnknown() { - // Given - Instant commitTime = Instant.parse("2024-08-14T09:58:01Z"); - List entries = List.of(commitAtTime(commitTime)); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)) - .contains(commitTime); - } - - @Test - void shouldGetNoTimeWhenNoRuns() { - // Given - List entries = List.of(); - - // When / Then - assertThat(StateStoreCommitterLogEntry.getLastTime(entries)).isEmpty(); - } - - private StateStoreCommitterRunStarted runStarted(Instant time) { - return new StateStoreCommitterRunStarted("test-stream", time); - } - - private StateStoreCommitterRunFinished runFinished(Instant time) { - return new StateStoreCommitterRunFinished("test-stream", time); - } - - private StateStoreCommitSummary commitAtTime(Instant time) { - return new StateStoreCommitSummary("test-stream", "test-table", "test-commit-type", time); - } -} diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java index 59ebfd43c6..6db041059d 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java @@ -17,91 +17,88 @@ import org.junit.jupiter.api.Test; +import sleeper.systemtest.dsl.testutil.drivers.InMemoryStateStoreCommitter; + import java.time.Instant; -import java.util.HashMap; import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; public class WaitForStateStoreCommitsTest { - Map waitForNumCommitsByTableId = new HashMap<>(); @Test void shouldFindOneCommitWasMadeAgainstCorrectTable() { // Given - waitForNumCommitsByTableId.put("test-table", 2); + List logs = List.of( + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:14:00Z"))); + Map waitForCommits = Map.of("test-table", 2); // When - WaitForStateStoreCommits.decrementWaitForNumCommits( - List.of(commitToTable("test-table")), - waitForNumCommitsByTableId); + Map remainingCommits = withLogs(logs).getRemainingCommitsInPeriod(waitForCommits, + Instant.parse("2024-08-14T12:00:00Z"), + Instant.parse("2024-08-14T13:00:00Z")); // Then - assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("test-table", 1)); + assertThat(remainingCommits).isEqualTo(Map.of("test-table", 1)); } @Test void shouldFindAllCommitsWereMadeAgainstCorrectTable() { // Given - waitForNumCommitsByTableId.put("test-table", 2); + List logs = List.of( + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:14:00Z")), + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:14:30Z"))); + Map waitForCommits = Map.of("test-table", 2); // When - WaitForStateStoreCommits.decrementWaitForNumCommits( - List.of(commitToTable("test-table"), commitToTable("test-table")), - waitForNumCommitsByTableId); + Map remainingCommits = withLogs(logs).getRemainingCommitsInPeriod(waitForCommits, + Instant.parse("2024-08-14T12:00:00Z"), + Instant.parse("2024-08-14T13:00:00Z")); // Then - assertThat(waitForNumCommitsByTableId).isEmpty(); + assertThat(remainingCommits).isEmpty(); } @Test void shouldFindCommitAgainstWrongTable() { // Given - waitForNumCommitsByTableId.put("test-table", 2); + List logs = List.of( + commitToTableAtTime("other-table", Instant.parse("2024-08-14T12:14:00Z"))); + Map waitForCommits = Map.of("test-table", 2); // When - WaitForStateStoreCommits.decrementWaitForNumCommits( - List.of(commitToTable("other-table")), - waitForNumCommitsByTableId); + Map remainingCommits = withLogs(logs).getRemainingCommitsInPeriod(waitForCommits, + Instant.parse("2024-08-14T12:00:00Z"), + Instant.parse("2024-08-14T13:00:00Z")); // Then - assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("test-table", 2)); + assertThat(remainingCommits).isEqualTo(Map.of("test-table", 2)); } @Test - void shouldFindCommitsAgainstMultipleTablesInOneRun() { + void shouldFindCommitsAgainstMultipleTables() throws Exception { // Given - waitForNumCommitsByTableId.put("table-1", 2); - waitForNumCommitsByTableId.put("table-2", 2); + List logs = List.of( + commitToTableAtTime("table-1", Instant.parse("2024-08-14T12:14:00Z")), + commitToTableAtTime("table-2", Instant.parse("2024-08-14T12:14:30Z"))); + Map waitForCommits = Map.of("table-1", 2, "table-2", 2); // When - WaitForStateStoreCommits.decrementWaitForNumCommits( - List.of(commitToTable("table-1"), commitToTable("table-2")), - waitForNumCommitsByTableId); + Map remainingCommits = withLogs(logs).getRemainingCommitsInPeriod(waitForCommits, + Instant.parse("2024-08-14T12:00:00Z"), + Instant.parse("2024-08-14T13:00:00Z")); - // Then - assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("table-1", 1, "table-2", 1)); + // When / Then + assertThat(remainingCommits).isEqualTo(Map.of("table-1", 1, "table-2", 1)); } - @Test - void shouldFindCommitsAgainstMultipleTablesInSeparateRuns() { - // Given - waitForNumCommitsByTableId.put("table-1", 2); - waitForNumCommitsByTableId.put("table-2", 2); - - // When - WaitForStateStoreCommits.decrementWaitForNumCommits(List.of( - commitToTable("table-1"), - commitToTable("table-2")), - waitForNumCommitsByTableId); - - // Then - assertThat(waitForNumCommitsByTableId).isEqualTo(Map.of("table-1", 1, "table-2", 1)); + private StateStoreCommitSummary commitToTableAtTime(String tableId, Instant time) { + return new StateStoreCommitSummary("test-stream", tableId, "test-commit-type", time); } - private StateStoreCommitSummary commitToTable(String tableId) { - return new StateStoreCommitSummary("test-stream", tableId, "test-commit-type", Instant.now()); + private WaitForStateStoreCommits withLogs(List logs) { + return new WaitForStateStoreCommits(InMemoryStateStoreCommitter.logsDriver(logs)); } } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java index d383a945b0..75bc72e780 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/InMemorySystemTestDrivers.java @@ -41,6 +41,7 @@ import sleeper.systemtest.dsl.sourcedata.GeneratedIngestSourceFilesDriver; import sleeper.systemtest.dsl.sourcedata.IngestSourceFilesDriver; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogsDriver; import sleeper.systemtest.dsl.testutil.drivers.InMemoryCompaction; import sleeper.systemtest.dsl.testutil.drivers.InMemoryDirectIngestDriver; import sleeper.systemtest.dsl.testutil.drivers.InMemoryDirectQueryDriver; @@ -114,6 +115,11 @@ public StateStoreCommitterDriver stateStoreCommitter(SystemTestContext context) return stateStoreCommitter.withContext(context); } + @Override + public StateStoreCommitterLogsDriver stateStoreCommitterLogs(SystemTestContext context) { + return stateStoreCommitter.logsDriver(); + } + @Override public DirectIngestDriver directIngest(SystemTestContext context) { return new InMemoryDirectIngestDriver(context.instance(), data, sketches); diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index 026614b852..0cecfa3bfc 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -27,6 +27,7 @@ import sleeper.systemtest.dsl.statestore.StateStoreCommitSummary; import sleeper.systemtest.dsl.statestore.StateStoreCommitterDriver; import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogEntry; +import sleeper.systemtest.dsl.statestore.StateStoreCommitterLogsDriver; import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunFinished; import sleeper.systemtest.dsl.statestore.StateStoreCommitterRunStarted; @@ -59,6 +60,10 @@ public StateStoreCommitterDriver withContext(SystemTestContext context) { return new Driver(context); } + public StateStoreCommitterLogsDriver logsDriver() { + return logsDriver(logs); + } + public void setRunCommitterOnSend(SleeperSystemTest sleeper, boolean runCommitterOnSend) { runCommitterOnSendByTableId.put(sleeper.tableProperties().get(TABLE_ID), runCommitterOnSend); } @@ -70,6 +75,13 @@ public void fakeRunWithCommits(SleeperSystemTest sleeper, int numCommits) { logs.add(new StateStoreCommitSummary("test-stream", tableId, "test-commit", Instant.now())); } + public static StateStoreCommitterLogsDriver logsDriver(List logs) { + return (startTime, endTime) -> logs.stream() + .filter(run -> run.getTimeInCommitter().compareTo(startTime) >= 0) + .filter(run -> run.getTimeInCommitter().compareTo(endTime) <= 0) + .collect(toUnmodifiableList()); + } + public class Driver implements StateStoreCommitterDriver { private final SystemTestInstanceContext instance; private final StateStoreCommitter committer; @@ -91,14 +103,6 @@ public void sendCommitMessages(Stream messages) { } } - @Override - public List getLogsInPeriod(Instant startTime, Instant endTime) { - return logs.stream() - .filter(run -> run.getTimeInCommitter().compareTo(startTime) >= 0) - .filter(run -> run.getTimeInCommitter().compareTo(endTime) <= 0) - .collect(toUnmodifiableList()); - } - private void runCommitter() { Instant startTime = Instant.now(); List commits = new ArrayList<>(); From 0a69a1eeffba9123f66947d5126cbf05b5af0261 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:37:55 +0000 Subject: [PATCH 35/49] Refactor WaitForStateStoreCommits --- .../statestore/WaitForStateStoreCommits.java | 18 ++++++---- .../WaitForStateStoreCommitsTest.java | 34 +++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java index 7e925bca1d..6cd6f1866f 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommits.java @@ -23,7 +23,6 @@ import java.time.Duration; import java.time.Instant; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.util.stream.Collectors.groupingBy; @@ -50,20 +49,24 @@ public void waitForCommits(PollWithRetries poll, Map waitForNum } public Map getRemainingCommitsInPeriod(Map waitForNumCommitsByTableId, Instant startTime, Instant endTime) { - Map remainingCommitsByTableId = new HashMap<>(waitForNumCommitsByTableId); - List logs = driver.getLogsInPeriod(startTime, endTime); - WaitForStateStoreCommits.decrementWaitForNumCommits(logs, remainingCommitsByTableId); + Map numCommitsByTableId = getNumCommitsByTableIdInPeriod(startTime, endTime); + Map remainingCommitsByTableId = getRemainingCommitsByTableId(waitForNumCommitsByTableId, numCommitsByTableId); LOGGER.info("Remaining unapplied commits by table ID: {}", remainingCommitsByTableId); return remainingCommitsByTableId; } - public static void decrementWaitForNumCommits(List entries, Map waitForNumCommitsByTableId) { - Map numCommitsByTableId = entries.stream() + private Map getNumCommitsByTableIdInPeriod(Instant startTime, Instant endTime) { + return driver.getLogsInPeriod(startTime, endTime).stream() .filter(entry -> entry instanceof StateStoreCommitSummary) .map(entry -> (StateStoreCommitSummary) entry) .collect(groupingBy(StateStoreCommitSummary::getTableId, summingInt(commit -> 1))); + } + + private static Map getRemainingCommitsByTableId( + Map waitForNumCommitsByTableId, Map numCommitsByTableId) { + Map remainingCommitsByTableId = new HashMap<>(waitForNumCommitsByTableId); numCommitsByTableId.forEach((tableId, numCommits) -> { - waitForNumCommitsByTableId.compute(tableId, (id, count) -> { + remainingCommitsByTableId.compute(tableId, (id, count) -> { if (count == null) { return null; } else if (numCommits >= count) { @@ -73,5 +76,6 @@ public static void decrementWaitForNumCommits(List } }); }); + return remainingCommitsByTableId; } } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java index 6db041059d..a727c1a92b 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java @@ -93,6 +93,40 @@ void shouldFindCommitsAgainstMultipleTables() throws Exception { assertThat(remainingCommits).isEqualTo(Map.of("table-1", 1, "table-2", 1)); } + @Test + void shouldIgnoreCommitBeforePeriod() throws Exception { + // Given + List logs = List.of( + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:14:00Z")), + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:16:00Z"))); + Map waitForCommits = Map.of("test-table", 2); + + // When + Map remainingCommits = withLogs(logs).getRemainingCommitsInPeriod(waitForCommits, + Instant.parse("2024-08-14T12:13:00Z"), + Instant.parse("2024-08-14T12:15:00Z")); + + // When / Then + assertThat(remainingCommits).isEqualTo(Map.of("test-table", 1)); + } + + @Test + void shouldIgnoreCommitAfterPeriod() throws Exception { + // Given + List logs = List.of( + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:14:00Z")), + commitToTableAtTime("test-table", Instant.parse("2024-08-14T12:16:00Z"))); + Map waitForCommits = Map.of("test-table", 2); + + // When + Map remainingCommits = withLogs(logs).getRemainingCommitsInPeriod(waitForCommits, + Instant.parse("2024-08-14T12:15:00Z"), + Instant.parse("2024-08-14T12:17:00Z")); + + // When / Then + assertThat(remainingCommits).isEqualTo(Map.of("test-table", 1)); + } + private StateStoreCommitSummary commitToTableAtTime(String tableId, Instant time) { return new StateStoreCommitSummary("test-stream", tableId, "test-commit-type", time); } From 93b900cb3d049e538c6d65db6cf5afd77e3f308c Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:42:45 +0000 Subject: [PATCH 36/49] Remove unused logger --- .../drivers/statestore/AwsStateStoreCommitterDriver.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java index 8642dd6b83..39a291e520 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/AwsStateStoreCommitterDriver.java @@ -18,8 +18,6 @@ import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.model.SendMessageBatchRequest; import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import sleeper.core.util.SplitIntoBatches; import sleeper.systemtest.dsl.instance.SystemTestInstanceContext; @@ -34,7 +32,6 @@ import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.STATESTORE_COMMITTER_QUEUE_URL; public class AwsStateStoreCommitterDriver implements StateStoreCommitterDriver { - public static final Logger LOGGER = LoggerFactory.getLogger(AwsStateStoreCommitterDriver.class); private final SystemTestInstanceContext instance; private final AmazonSQS sqs; From 9e2232a16e73a8538e95c7b695e5f3b8009206fa Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:45:17 +0000 Subject: [PATCH 37/49] Remove unused code --- .../ReadStateStoreCommitterLogs.java | 75 +------------------ 1 file changed, 3 insertions(+), 72 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java index 6f97533683..d8678017af 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java @@ -30,6 +30,9 @@ public class ReadStateStoreCommitterLogs { + private ReadStateStoreCommitterLogs() { + } + private static final Pattern MESSAGE_PATTERN = Pattern.compile("" + "Lambda started at (.+)|" + // Lambda started message type "Lambda finished at ([^ ]+) |" + // Lambda finished message type @@ -46,9 +49,6 @@ private CapturingGroups() { } } - private ReadStateStoreCommitterLogs() { - } - private static StateStoreCommitterLogEntry readMessage(String logStream, String message) { Matcher matcher = MESSAGE_PATTERN.matcher(message); if (!matcher.find()) { @@ -93,73 +93,4 @@ public static StateStoreCommitterLogEntry read(List entry) { Objects.requireNonNull(logStream, "Log stream not found"), Objects.requireNonNull(message, "Log message not found")); } - - public static class LambdaStarted { - private final Instant startTime; - - public LambdaStarted(Instant startTime) { - this.startTime = startTime; - } - - public Instant getStartTime() { - return startTime; - } - - @Override - public int hashCode() { - return Objects.hash(startTime); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof LambdaStarted)) { - return false; - } - LambdaStarted other = (LambdaStarted) obj; - return Objects.equals(startTime, other.startTime); - } - - @Override - public String toString() { - return "LambdaStarted{startTime=" + startTime + "}"; - } - } - - public static class LambdaFinished { - private final Instant finishTime; - - public LambdaFinished(Instant finishTime) { - this.finishTime = finishTime; - } - - public Instant getFinishTime() { - return finishTime; - } - - @Override - public int hashCode() { - return Objects.hash(finishTime); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof LambdaFinished)) { - return false; - } - LambdaFinished other = (LambdaFinished) obj; - return Objects.equals(finishTime, other.finishTime); - } - - @Override - public String toString() { - return "LambdaFinished{finishTime=" + finishTime + "}"; - } - } - } From ad89a135db81ca7ebe146e6e8dcbd61477b6b66f Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:47:15 +0000 Subject: [PATCH 38/49] Remove extra helper from ReadStateStoreCommitterLogsTest --- .../statestore/ReadStateStoreCommitterLogsTest.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java index 75d3314f11..f89484b4f9 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java @@ -72,9 +72,12 @@ void shouldReadUnrecognisedLog() { void shouldReadLogWithFieldsInReverseOrder() { // Given String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + List entry = List.of( + ResultField.builder().field("@message").value(message).build(), + ResultField.builder().field("@logStream").value("test-stream").build()); // When / Then - assertThat(ReadStateStoreCommitterLogs.read(logEntryReversed("test-stream", message))).isEqualTo( + assertThat(ReadStateStoreCommitterLogs.read(entry)).isEqualTo( new StateStoreCommitterRunStarted("test-stream", Instant.parse("2024-08-13T12:12:00Z"))); } @@ -99,10 +102,4 @@ private List logEntry(String logStream, String message) { ResultField.builder().field("@logStream").value(logStream).build(), ResultField.builder().field("@message").value(message).build()); } - - private List logEntryReversed(String logStream, String message) { - return List.of( - ResultField.builder().field("@message").value(message).build(), - ResultField.builder().field("@logStream").value(logStream).build()); - } } From 8c6b58e7951073e6cb38ea878bbb03285585e24c Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:47:55 +0000 Subject: [PATCH 39/49] Remove unused method --- .../dsl/statestore/StateStoreCommitterLogEntry.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java index 83de134f08..f149712a47 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/StateStoreCommitterLogEntry.java @@ -16,19 +16,10 @@ package sleeper.systemtest.dsl.statestore; import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; public interface StateStoreCommitterLogEntry { String getLogStream(); Instant getTimeInCommitter(); - - static Optional getLastTime(List entries) { - return entries.stream() - .map(StateStoreCommitterLogEntry::getTimeInCommitter) - .max(Comparator.naturalOrder()); - } } From c4b8b6f0d00d1891de7586a72d67ffbb9b54f37a Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:48:25 +0000 Subject: [PATCH 40/49] Remove unused logger --- .../dsl/statestore/SystemTestStateStoreFakeCommits.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index 53967048ae..1c2c03274d 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -15,9 +15,6 @@ */ package sleeper.systemtest.dsl.statestore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SystemTestContext; import sleeper.systemtest.dsl.SystemTestDrivers; @@ -30,7 +27,6 @@ import java.util.stream.Stream; public class SystemTestStateStoreFakeCommits { - public static final Logger LOGGER = LoggerFactory.getLogger(SystemTestStateStoreFakeCommits.class); private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; From 74c49b74ec2b466aabd6b197d98496055fb6eaff Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:51:50 +0000 Subject: [PATCH 41/49] Fix SystemTestStateStoreFakeCommitsTest --- .../statestore/SystemTestStateStoreFakeCommitsTest.java | 6 ++++-- .../dsl/testutil/drivers/InMemoryStateStoreCommitter.java | 7 ++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index 6543f0a6fc..90482daf74 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -24,10 +24,12 @@ import sleeper.systemtest.dsl.testutil.InMemorySystemTestDrivers; import sleeper.systemtest.dsl.testutil.drivers.InMemoryStateStoreCommitter; +import java.time.Instant; import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import static sleeper.configuration.properties.table.TableProperty.TABLE_ID; import static sleeper.core.testutils.printers.FileReferencePrinter.printFiles; import static sleeper.systemtest.dsl.testutil.InMemoryTestInstance.MAIN; @@ -70,12 +72,12 @@ void shouldSendManyFileCommits(SleeperSystemTest sleeper) throws Exception { } @Test - void shouldWaitForCommitWhenCommitWasMadeButRunIsUnfinished(SleeperSystemTest sleeper) throws Exception { + void shouldWaitForCommitWhenCommitWasMadeButNoRunStartOrFinishLogsWereMade(SleeperSystemTest sleeper) throws Exception { // Given committer.setRunCommitterOnSend(sleeper, false); SystemTestStateStoreFakeCommits commitsDsl = sleeper.stateStore().fakeCommits(); commitsDsl.send(factory -> factory.addPartitionFile("root", "file.parquet", 100)); - committer.fakeRunWithCommits(sleeper, 1); + committer.addFakeLog(new StateStoreCommitSummary("test-stream", sleeper.tableProperties().get(TABLE_ID), "test-commit", Instant.now())); // When / Then assertThatCode(() -> commitsDsl.waitForCommits(PollWithRetries.noRetries())) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java index 0cecfa3bfc..4c309d91e0 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/testutil/drivers/InMemoryStateStoreCommitter.java @@ -68,11 +68,8 @@ public void setRunCommitterOnSend(SleeperSystemTest sleeper, boolean runCommitte runCommitterOnSendByTableId.put(sleeper.tableProperties().get(TABLE_ID), runCommitterOnSend); } - public void fakeRunWithCommits(SleeperSystemTest sleeper, int numCommits) { - String tableId = sleeper.tableProperties().get(TABLE_ID); - logs.add(new StateStoreCommitterRunStarted("test-stream", Instant.now())); - logs.add(new StateStoreCommitterRunFinished("test-stream", Instant.now())); - logs.add(new StateStoreCommitSummary("test-stream", tableId, "test-commit", Instant.now())); + public void addFakeLog(StateStoreCommitterLogEntry entry) { + logs.add(entry); } public static StateStoreCommitterLogsDriver logsDriver(List logs) { From 8807267227d2b3630d410c065b1608d57fdce291 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:08:28 +0000 Subject: [PATCH 42/49] Set skipRust property in deployTest.sh --- scripts/test/maven/deployTest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/maven/deployTest.sh b/scripts/test/maven/deployTest.sh index 6200b3ab31..9e5beaf997 100755 --- a/scripts/test/maven/deployTest.sh +++ b/scripts/test/maven/deployTest.sh @@ -37,7 +37,7 @@ START_TIME=$(record_time) pushd "$MAVEN_DIR" -mvn verify -PsystemTest \ +mvn verify -PsystemTest -DskipRust=true \ -Dsleeper.system.test.short.id="$SHORT_ID" \ -Dsleeper.system.test.vpc.id="$VPC" \ -Dsleeper.system.test.subnet.ids="$SUBNETS" \ From 05d63664d834f98992b0b448882bae010d043981 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:33:41 +0000 Subject: [PATCH 43/49] Grant StartQuery/GetQueryResults for log stream --- .../main/java/sleeper/cdk/stack/StateStoreCommitterStack.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java index 22afd17678..1da61fcee9 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/StateStoreCommitterStack.java @@ -136,6 +136,7 @@ private void lambdaToCommitStateStoreUpdates( .build()); logGroup.grantRead(policiesStack.getReportingPolicyForGrants()); + logGroup.grant(policiesStack.getReportingPolicyForGrants(), "logs:StartQuery", "logs:GetQueryResults"); configBucketStack.grantRead(handlerFunction); tableIndexStack.grantRead(handlerFunction); stateStoreStacks.grantReadWriteAllFilesAndPartitions(handlerFunction); From c5085710d6cd4fe3381e9ffc319ed54c4a004838 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:18:16 +0000 Subject: [PATCH 44/49] Handle new line at end of log message --- .../statestore/ReadStateStoreCommitterLogs.java | 6 +++--- .../statestore/ReadStateStoreCommitterLogsTest.java | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java index d8678017af..9e5e6f0623 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogs.java @@ -34,9 +34,9 @@ private ReadStateStoreCommitterLogs() { } private static final Pattern MESSAGE_PATTERN = Pattern.compile("" + - "Lambda started at (.+)|" + // Lambda started message type - "Lambda finished at ([^ ]+) |" + // Lambda finished message type - "Applied request to table ID ([^ ]+) with type ([^ ]+) at time ([^ ]+)"); // Commit applied message type + "Lambda started at ([^\\s]+)|" + // Lambda started message type + "Lambda finished at ([^\\s]+) |" + // Lambda finished message type + "Applied request to table ID ([^\\s]+) with type ([^\\s]+) at time ([^\\s]+)"); // Commit applied message type private static class CapturingGroups { private static final int START_TIME = 1; diff --git a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java index f89484b4f9..f83e11695d 100644 --- a/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java +++ b/java/system-test/system-test-drivers/src/test/java/sleeper/systemtest/drivers/statestore/ReadStateStoreCommitterLogsTest.java @@ -32,7 +32,7 @@ public class ReadStateStoreCommitterLogsTest { @Test void shouldReadLambdaStarted() { // Given - String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z\n"; // When / Then assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isEqualTo( @@ -42,7 +42,7 @@ void shouldReadLambdaStarted() { @Test void shouldReadLambdaFinished() { // Given - String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)"; + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda finished at 2024-08-13T12:13:00Z (ran for 1 minute)\n"; // When / Then assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isEqualTo( @@ -52,7 +52,7 @@ void shouldReadLambdaFinished() { @Test void shouldReadCommitApplied() { // Given - String message = "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z"; + String message = "[main] sleeper.commit.StateStoreCommitter INFO - Applied request to table ID test-table with type TestRequest at time 2024-08-13T12:12:30Z\n"; // When / Then assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isEqualTo( @@ -62,7 +62,7 @@ void shouldReadCommitApplied() { @Test void shouldReadUnrecognisedLog() { // Given - String message = "some other log"; + String message = "some other log\n"; // When / Then assertThat(ReadStateStoreCommitterLogs.read(logEntry("test-stream", message))).isNull(); @@ -71,7 +71,7 @@ void shouldReadUnrecognisedLog() { @Test void shouldReadLogWithFieldsInReverseOrder() { // Given - String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z\n"; List entry = List.of( ResultField.builder().field("@message").value(message).build(), ResultField.builder().field("@logStream").value("test-stream").build()); @@ -84,7 +84,7 @@ void shouldReadLogWithFieldsInReverseOrder() { @Test void shouldReadLogWithExtraFields() { // Given - String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z"; + String message = "[main] committer.lambda.StateStoreCommitterLambda INFO - Lambda started at 2024-08-13T12:12:00Z\n"; List entry = List.of( ResultField.builder().field("@extraField").value("some value").build(), ResultField.builder().field("@message").value(message).build(), From c19975bcd2491f9d0e1a0649fcce35ac3919cd01 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:26:04 +0000 Subject: [PATCH 45/49] Adjust assertions in StateStoreCommitterST --- .../java/sleeper/systemtest/suite/StateStoreCommitterST.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java index 1b60707bed..c758b91afd 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -53,11 +53,11 @@ void shouldAddManyFiles(SleeperSystemTest sleeper) throws Exception { .sendBatched(commitFactory -> IntStream.rangeClosed(1, 1000) .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) .map(commitFactory::addFile)) - .waitForCommits(PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(5), Duration.ofMinutes(2))); + .waitForCommits(PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(20), Duration.ofMinutes(3))); // Then assertThat(sleeper.tableFiles().references()) - .containsExactlyElementsOf(IntStream.rangeClosed(1, 1000) + .containsExactlyInAnyOrderElementsOf(IntStream.rangeClosed(1, 1000) .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) .collect(toUnmodifiableList())); } From b2795a8e2bba5383f91defc6f1670941a5287fb7 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:27:57 +0000 Subject: [PATCH 46/49] Rename WaitForStateStoreCommitLogs --- .../dsl/statestore/SystemTestStateStoreFakeCommits.java | 8 ++++---- ...StoreCommits.java => WaitForStateStoreCommitLogs.java} | 8 ++++---- .../statestore/SystemTestStateStoreFakeCommitsTest.java | 6 +++--- .../dsl/statestore/WaitForStateStoreCommitsTest.java | 4 ++-- .../sleeper/systemtest/suite/StateStoreCommitterST.java | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) rename java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/{WaitForStateStoreCommits.java => WaitForStateStoreCommitLogs.java} (91%) diff --git a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java index 1c2c03274d..7d3e9f4162 100644 --- a/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java +++ b/java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommits.java @@ -30,7 +30,7 @@ public class SystemTestStateStoreFakeCommits { private final SystemTestInstanceContext instance; private final StateStoreCommitterDriver driver; - private final WaitForStateStoreCommits waiter; + private final WaitForStateStoreCommitLogs waiter; private final Map waitForNumCommitsByTableId = new ConcurrentHashMap<>(); private final Instant getRunsAfterTime; @@ -38,7 +38,7 @@ public SystemTestStateStoreFakeCommits(SystemTestContext context) { instance = context.instance(); SystemTestDrivers adminDrivers = context.instance().adminDrivers(); driver = adminDrivers.stateStoreCommitter(context); - waiter = new WaitForStateStoreCommits(adminDrivers.stateStoreCommitterLogs(context)); + waiter = new WaitForStateStoreCommitLogs(adminDrivers.stateStoreCommitterLogs(context)); getRunsAfterTime = context.reporting().getRecordingStartTime(); } @@ -52,8 +52,8 @@ public SystemTestStateStoreFakeCommits send(Function waitForNumCommitsByTableId, Instant getRunsAfterTime) throws InterruptedException { + public void waitForCommitLogs(PollWithRetries poll, Map waitForNumCommitsByTableId, Instant getRunsAfterTime) throws InterruptedException { LOGGER.info("Waiting for commits by table ID: {}", waitForNumCommitsByTableId); Instant startTime = getRunsAfterTime.minus(QUERY_RUNS_TIME_SLACK); poll.pollUntil("all state store commits are applied", () -> { diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java index 90482daf74..ba41240210 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/SystemTestStateStoreFakeCommitsTest.java @@ -49,7 +49,7 @@ void shouldSendOneFileCommit(SleeperSystemTest sleeper) throws Exception { // When sleeper.stateStore().fakeCommits() .send(factory -> factory.addPartitionFile("root", "file.parquet", 100)) - .waitForCommits(PollWithRetries.noRetries()); + .waitForCommitLogs(PollWithRetries.noRetries()); // Then assertThat(printFiles(sleeper.partitioning().tree(), sleeper.tableFiles().all())) @@ -65,7 +65,7 @@ void shouldSendManyFileCommits(SleeperSystemTest sleeper) throws Exception { sleeper.stateStore().fakeCommits() .sendBatched(factory -> LongStream.rangeClosed(1, 1000) .mapToObj(i -> factory.addPartitionFile("root", "file-" + i + ".parquet", i))) - .waitForCommits(PollWithRetries.noRetries()); + .waitForCommitLogs(PollWithRetries.noRetries()); // Then assertThat(sleeper.tableFiles().references()).hasSize(1000); @@ -80,7 +80,7 @@ void shouldWaitForCommitWhenCommitWasMadeButNoRunStartOrFinishLogsWereMade(Sleep committer.addFakeLog(new StateStoreCommitSummary("test-stream", sleeper.tableProperties().get(TABLE_ID), "test-commit", Instant.now())); // When / Then - assertThatCode(() -> commitsDsl.waitForCommits(PollWithRetries.noRetries())) + assertThatCode(() -> commitsDsl.waitForCommitLogs(PollWithRetries.noRetries())) .doesNotThrowAnyException(); } diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java index a727c1a92b..bfae367c23 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/statestore/WaitForStateStoreCommitsTest.java @@ -131,8 +131,8 @@ private StateStoreCommitSummary commitToTableAtTime(String tableId, Instant time return new StateStoreCommitSummary("test-stream", tableId, "test-commit-type", time); } - private WaitForStateStoreCommits withLogs(List logs) { - return new WaitForStateStoreCommits(InMemoryStateStoreCommitter.logsDriver(logs)); + private WaitForStateStoreCommitLogs withLogs(List logs) { + return new WaitForStateStoreCommitLogs(InMemoryStateStoreCommitter.logsDriver(logs)); } } diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java index c758b91afd..a49aec3a68 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -53,7 +53,7 @@ void shouldAddManyFiles(SleeperSystemTest sleeper) throws Exception { .sendBatched(commitFactory -> IntStream.rangeClosed(1, 1000) .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) .map(commitFactory::addFile)) - .waitForCommits(PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(20), Duration.ofMinutes(3))); + .waitForCommitLogs(PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(20), Duration.ofMinutes(3))); // Then assertThat(sleeper.tableFiles().references()) From ca9b8f1fecda785db2e52a83b3d10a4ece7d907b Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:34:53 +0000 Subject: [PATCH 47/49] Ignore file update time in assertion --- .../java/sleeper/systemtest/suite/StateStoreCommitterST.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java index a49aec3a68..482ade59f3 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -57,6 +57,7 @@ void shouldAddManyFiles(SleeperSystemTest sleeper) throws Exception { // Then assertThat(sleeper.tableFiles().references()) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("lastStateStoreUpdateTime") .containsExactlyInAnyOrderElementsOf(IntStream.rangeClosed(1, 1000) .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) .collect(toUnmodifiableList())); From 3064b3688c771f8c16b01ad99c5eac12523ffea8 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:40:23 +0000 Subject: [PATCH 48/49] Adjust assertion on files --- .../systemtest/suite/StateStoreCommitterST.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java index 482ade59f3..96de25ee44 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -20,15 +20,17 @@ import sleeper.core.partition.PartitionTree; import sleeper.core.partition.PartitionsBuilder; +import sleeper.core.statestore.FileReference; import sleeper.core.statestore.FileReferenceFactory; import sleeper.core.util.PollWithRetries; import sleeper.systemtest.dsl.SleeperSystemTest; import sleeper.systemtest.suite.testutil.SystemTest; import java.time.Duration; +import java.util.Map; import java.util.stream.IntStream; -import static java.util.stream.Collectors.toUnmodifiableList; +import static java.util.stream.Collectors.toMap; import static org.assertj.core.api.Assertions.assertThat; import static sleeper.systemtest.suite.fixtures.SystemTestInstance.MAIN; import static sleeper.systemtest.suite.fixtures.SystemTestSchema.DEFAULT_SCHEMA; @@ -56,10 +58,10 @@ void shouldAddManyFiles(SleeperSystemTest sleeper) throws Exception { .waitForCommitLogs(PollWithRetries.intervalAndPollingTimeout(Duration.ofSeconds(20), Duration.ofMinutes(3))); // Then - assertThat(sleeper.tableFiles().references()) - .usingRecursiveFieldByFieldElementComparatorIgnoringFields("lastStateStoreUpdateTime") - .containsExactlyInAnyOrderElementsOf(IntStream.rangeClosed(1, 1000) - .mapToObj(i -> fileFactory.rootFile("file-" + i + ".parquet", i)) - .collect(toUnmodifiableList())); + Map recordsByFilename = sleeper.tableFiles().references().stream() + .collect(toMap(FileReference::getFilename, FileReference::getNumberOfRecords)); + assertThat(recordsByFilename).isEqualTo( + IntStream.rangeClosed(1, 1000).mapToObj(i -> i) + .collect(toMap(i -> "file-" + i + ".parquet", i -> i))); } } From 48958ad10c3123dd7340eb8736e8c79c0224619b Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:44:04 +0000 Subject: [PATCH 49/49] Use long for record count in assertion --- .../java/sleeper/systemtest/suite/StateStoreCommitterST.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java index 96de25ee44..5ab0749ed5 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterST.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.util.Map; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static java.util.stream.Collectors.toMap; import static org.assertj.core.api.Assertions.assertThat; @@ -61,7 +62,7 @@ void shouldAddManyFiles(SleeperSystemTest sleeper) throws Exception { Map recordsByFilename = sleeper.tableFiles().references().stream() .collect(toMap(FileReference::getFilename, FileReference::getNumberOfRecords)); assertThat(recordsByFilename).isEqualTo( - IntStream.rangeClosed(1, 1000).mapToObj(i -> i) + LongStream.rangeClosed(1, 1000).mapToObj(i -> i) .collect(toMap(i -> "file-" + i + ".parquet", i -> i))); } }