From dd02a7980b8cc48669271952189241ce489a2278 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 24 Dec 2024 17:39:33 +0000 Subject: [PATCH 1/7] WIP: run tests in parallel This begins the process of allowing for parallel test execution. A few issues were noticed that required some changes: * Some tests were taking up a bunch of time and benefitted from parallelized execution of tests within the fixture. Those have been updated to CONCURRENT execution mode * A known issue with JUnit (https://github.com/junit-team/junit5/issues/3108) means that if one of the tests involves a future waiting, that can look to the `ForkJoinPool` like the thread is available for work stealing, so too many tests can end up being executed at once. A new test extension was added that has a semaphore, and that appears to be enough to stop extra tests from being executed * The server was running out of batch GRV transactions, which resulted in tests failing with "batch GRV transactions exhausted". This mainly affected indexing tests, and I was able to resolve this by upping the transaction priority, for better or worse There are still some issues: * A number of tests were hitting deadline exceeded exceptions. It looked like some kind of weird concurrency stuff may be going on, because there were things happening like key space path resolution while creating a record store would include stack traces from closing the test key space path manager, which seems like things were sharing objects that shouldn't have been. This affected both the `:fdb-record-layer-core:test` and `:fdb-lucene:test` tasks * The relational layer tests are not structured to allocate unique key spaces for each test, so they immediatetly hit concurrency problems when run in a parallelized manner --- .../apple/test/LimitConcurrencyExtension.java | 44 +++++++ .../FDBRecordStoreCountRecordsTest.java | 5 +- .../foundationdb/FDBRecordStoreIndexTest.java | 3 +- .../foundationdb/FDBRecordStoreTestBase.java | 14 +++ .../FDBRecordStoreUniqueIndexTest.java | 11 +- .../OnlineIndexerBuildUnnestedIndexTest.java | 1 + .../foundationdb/OnlineIndexerMergeTest.java | 1 + .../foundationdb/OnlineIndexerTest.java | 2 + .../foundationdb/RecordTypeKeyTest.java | 6 +- .../SimpleMultidimensionalIndexTest.java | 113 +++++++++--------- .../SlowMultidimensionalIndexTest.java | 39 +++--- .../indexes/VersionIndexTest.java | 3 + .../query/FDBOrQueryToUnionTest.java | 3 + .../lucene/FDBLuceneIndexFailureTest.java | 3 +- .../lucene/LuceneIndexMaintenanceTest.java | 5 + .../record/lucene/LuceneIndexTest.java | 3 + .../lucene/LuceneOnlineIndexingTest.java | 69 ++++------- .../lucene/directory/AgilityContextTest.java | 37 +++--- gradle/testing.gradle | 4 +- 19 files changed, 208 insertions(+), 158 deletions(-) create mode 100644 fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java diff --git a/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java b/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java new file mode 100644 index 0000000000..53245ba67b --- /dev/null +++ b/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java @@ -0,0 +1,44 @@ +/* + * LimitConcurrencyExtension.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.test; + +import com.google.auto.service.AutoService; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.util.concurrent.Semaphore; + +@AutoService(Extension.class) +public class LimitConcurrencyExtension implements BeforeEachCallback, AfterEachCallback { + static Semaphore testConcurrency = new Semaphore(Integer.parseInt(System.getProperty("tests.concurrencyLimit", "10"))); + + @Override + public void beforeEach(final ExtensionContext context) throws InterruptedException { + testConcurrency.acquire(); + } + + @Override + public void afterEach(final ExtensionContext context) { + testConcurrency.release(); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreCountRecordsTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreCountRecordsTest.java index d0c969ba42..65eb4ae1a0 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreCountRecordsTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreCountRecordsTest.java @@ -45,6 +45,8 @@ import com.google.protobuf.Message; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import javax.annotation.Nonnull; import java.util.HashMap; @@ -69,6 +71,7 @@ * Tests related to built in functionality for getting the count of records in a store. */ @Tag(Tags.RequiresFDB) +@Execution(ExecutionMode.CONCURRENT) public class FDBRecordStoreCountRecordsTest extends FDBRecordStoreTestBase { @Test @@ -380,7 +383,7 @@ public void addCountIndex() throws Exception { } // Build the index - try (OnlineIndexer onlineIndexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "record_count")) { + try (OnlineIndexer onlineIndexBuilder = newIndexer("record_count")) { onlineIndexBuilder.buildIndex(); } try (FDBRecordContext context = openContext()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java index af3f89a54c..7f8738a07a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java @@ -1239,8 +1239,7 @@ void markReadableTest() throws Exception { openSimpleRecordStore(context); Index index = recordStore.getRecordMetaData().getIndex(indexName); assertThat(recordStore.isIndexReadable(index), is(false)); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore).setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(indexName)) { indexBuilder.buildIndex(false); } commit(context); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTestBase.java index c7bc419919..19a935c702 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTestBase.java @@ -305,4 +305,18 @@ protected FDBStoredRecord saveAndSplitSimpleRecord(long recno, String s md.removeIndex("MySimpleRecord$str_value_indexed"); md.setStoreRecordVersions(false); }; + + @Nonnull + protected OnlineIndexer.Builder newIndexerBuilder() { + return OnlineIndexer.newBuilder() + .setRecordStore(recordStore) + .setPriority(FDBTransactionPriority.DEFAULT); + } + + @Nonnull + protected OnlineIndexer newIndexer(@Nonnull String indexName) { + return newIndexerBuilder() + .setIndex(indexName) + .build(); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreUniqueIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreUniqueIndexTest.java index cd6eeaf48c..60c14765fb 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreUniqueIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreUniqueIndexTest.java @@ -45,8 +45,8 @@ import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.indexes.InvalidIndexEntry; import com.apple.foundationdb.record.provider.foundationdb.indexes.ValueIndexMaintainer; -import com.apple.foundationdb.record.query.QueryToKeyMatcher; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath; +import com.apple.foundationdb.record.query.QueryToKeyMatcher; import com.apple.foundationdb.record.query.plan.QueryPlanner; import com.apple.foundationdb.record.test.TestKeySpace; import com.apple.foundationdb.record.util.pair.Pair; @@ -59,6 +59,8 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -98,6 +100,7 @@ * Tests of uniqueness checks. */ @Tag(Tags.RequiresFDB) +@Execution(ExecutionMode.CONCURRENT) public class FDBRecordStoreUniqueIndexTest extends FDBRecordStoreTestBase { private static final String NO_UNIQUE_CLEAR_INDEX_TYPE = "no_unique_clear"; @@ -310,8 +313,7 @@ void multipleStores() throws Exception { recordStore = createOrOpenRecordStore(context, simpleMetaData(uniqueHook), path).getLeft(); commit(context); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setTargetIndexes(List.of(uniqueIndex)) .setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder() .allowUniquePendingState(true)) @@ -630,8 +632,7 @@ public void addUniqueIndexViaBuild() { if (allowReadableUniquePending) { indexingPolicy.allowUniquePendingState(); } - try (OnlineIndexer indexer = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexer = newIndexerBuilder() .setTargetIndexesByName(List.of(uniqueIndex.getName())) .setIndexingPolicy(indexingPolicy .build()).build()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerBuildUnnestedIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerBuildUnnestedIndexTest.java index 687cf7e41d..243b067ac6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerBuildUnnestedIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerBuildUnnestedIndexTest.java @@ -679,6 +679,7 @@ void doNotAllowBuildingIndexFromUnnestedIndex() { try (OnlineIndexer indexer = OnlineIndexer.newBuilder() .setRecordStoreBuilder(storeBuilder) .setIndex(targetIndex) + .setPriority(FDBTransactionPriority.DEFAULT) .setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder() .setSourceIndex(sourceIndex.getName()) .setIfDisabled(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java index 4aded5dcc7..37b42aa636 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java @@ -145,6 +145,7 @@ void testRepartitionTimeout() { try (OnlineIndexer indexer = OnlineIndexer.newBuilder() .setRecordStoreBuilder(storeBuilder) .setTargetIndexesByName(List.of(INDEX_NAME)) + .setPriority(FDBTransactionPriority.DEFAULT) .setMaxAttempts(9) .build()) { Assertions.assertThrows(FDBExceptions.FDBStoreTransactionIsTooOldException.class, indexer::mergeIndex); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java index 6e5824c1cb..6f9bf4cda3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java @@ -156,6 +156,7 @@ OnlineIndexer.Builder newIndexerBuilder() { .setMetaData(metaData) .setSubspaceProvider(new SubspaceProviderByKeySpacePath(path)) .setIndexMaintenanceFilter(getIndexMaintenanceFilter()) + .setPriority(FDBTransactionPriority.DEFAULT) .setFormatVersion(formatVersion); } @@ -181,6 +182,7 @@ OnlineIndexScrubber.Builder newScrubberBuilder() { .setMetaData(metaData) .setSubspaceProvider(new SubspaceProviderByKeySpacePath(path)) .setIndexMaintenanceFilter(getIndexMaintenanceFilter()) + .setPriority(FDBTransactionPriority.DEFAULT) .setFormatVersion(formatVersion); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java index 46aec50a58..cce49140c1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java @@ -602,7 +602,7 @@ public void testOnlineIndexBuilder() throws Exception { timer.reset(); // Build in this transaction. - try (OnlineIndexer indexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "newIndex")) { + try (OnlineIndexer indexBuilder = newIndexer("newIndex")) { indexBuilder.rebuildIndex(recordStore); } recordStore.markIndexReadable("newIndex").join(); @@ -629,7 +629,7 @@ public void testOnlineIndexBuilder() throws Exception { timer.reset(); // Build in multiple transactions. - try (OnlineIndexer indexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "newIndex")) { + try (OnlineIndexer indexBuilder = newIndexer("newIndex")) { indexBuilder.buildIndex(); } @@ -682,7 +682,7 @@ public void testOnlineIndexMultiTargetBuilder() throws Exception { } // Build multiple indexes of typed records - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .addTargetIndex("newIndex") .addTargetIndex("newSumIndex") .addTargetIndex("newMaxIndex") diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java index f203aa1d04..9296664df2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java @@ -21,6 +21,8 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -35,18 +37,22 @@ /** * Simple tests for Multidimensional Index. */ +@Execution(ExecutionMode.CONCURRENT) class SimpleMultidimensionalIndexTest extends MultidimensionalIndexTestBase { + static Stream storageAdapterArgs() { + return Stream.of(BY_NODE.toString(), BY_SLOT.toString()); + } + + static Stream booleanArgs() { + return Stream.of(false, true); + } + static Stream argumentsForBasicReads() { - return Stream.of( - Arguments.of(BY_NODE.toString(), false, false), - Arguments.of(BY_NODE.toString(), false, true), - Arguments.of(BY_NODE.toString(), true, false), - Arguments.of(BY_NODE.toString(), true, true), - Arguments.of(BY_SLOT.toString(), false, false), - Arguments.of(BY_SLOT.toString(), false, true), - Arguments.of(BY_SLOT.toString(), true, false), - Arguments.of(BY_SLOT.toString(), true, true)); + return storageAdapterArgs().flatMap(storageAdapter -> + booleanArgs().flatMap(storeHilbertValues -> + booleanArgs().map(useNodeSlotIndex -> + Arguments.of(storageAdapter, storeHilbertValues, useNodeSlotIndex)))); } /** @@ -57,62 +63,51 @@ static Stream argumentsForBasicReads() { * * @return a stream of arguments */ + @Nonnull static Stream argumentsForIndexReads() { final Random random = new Random(System.currentTimeMillis()); - return Stream.of( - Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 100, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 100, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 100, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 100, BY_NODE.toString(), true, true), - Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 500, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 500, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 500, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 500, BY_NODE.toString(), true, true), - // large values only for default config - Arguments.of(random.nextLong(), 1000, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 5000, BY_NODE.toString(), true, false) - ); + return storageAdapterArgs().flatMap(storageAdapter -> + booleanArgs().flatMap(storeHilbertValue -> + booleanArgs().flatMap(useNodeSlotIndex -> + argumentsForIndexReads(random, storageAdapter, storeHilbertValue, useNodeSlotIndex)))); } + @Nonnull + static Stream argumentsForIndexReads(@Nonnull Random random, @Nonnull String storageAdapter, boolean storeHilbertValue, boolean useNodeSlotIndex) { + Arguments small = Arguments.of(random.nextLong(), 100, storageAdapter, storeHilbertValue, useNodeSlotIndex); + Arguments medium = Arguments.of(random.nextLong(), 500, storageAdapter, storeHilbertValue, useNodeSlotIndex); + // large values only for default config + if (storeHilbertValue && !useNodeSlotIndex && storageAdapter.equals(BY_NODE.toString())) { + Arguments large = Arguments.of(random.nextLong(), 1000, storageAdapter, storeHilbertValue, useNodeSlotIndex); + Arguments extraLarge = Arguments.of(random.nextLong(), 5000, storageAdapter, storeHilbertValue, useNodeSlotIndex); + return Stream.of(small, medium, large, extraLarge); + } else { + return Stream.of(small, medium); + } + } + + @Nonnull static Stream argumentsForIndexReadsAfterDeletes() { final Random random = new Random(System.currentTimeMillis()); - return Stream.of( - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), true, true), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), true, true), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), true, true), - // large values only for default config - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 5000, random.nextInt(1000) + 1, BY_NODE.toString(), true, false) - ); + return storageAdapterArgs().flatMap(storageAdapter -> + booleanArgs().flatMap(storeHilbertValue -> + booleanArgs().flatMap(useNodeSlotIndex -> + argumentsForIndexReadsAfterDeletes(random, storageAdapter, storeHilbertValue, useNodeSlotIndex)))); + } + + @Nonnull + static Stream argumentsForIndexReadsAfterDeletes(@Nonnull Random random, @Nonnull String storageAdapter, boolean storeHilbertValue, boolean useNodeSlotIndex) { + Arguments extraSmall = Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex); + Arguments small = Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex); + Arguments medium = Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex); + // large values only for default config + if (storeHilbertValue && !useNodeSlotIndex && storageAdapter.equals(BY_NODE.toString())) { + Arguments large = Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex); + Arguments extraLarge = Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex); + return Stream.of(extraSmall, small, medium, large, extraLarge); + } else { + return Stream.of(extraSmall, small, medium); + } } /** diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java index 2ca83d6c6b..9ab579ecdf 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java @@ -25,12 +25,15 @@ import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; +import com.apple.test.RandomizedTestUtils; import com.apple.test.Tags; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -49,6 +52,7 @@ * Additional tests for Multidimensional Index around concurrency and large data-sets. */ @Tag(Tags.Slow) +@Execution(ExecutionMode.CONCURRENT) class SlowMultidimensionalIndexTest extends MultidimensionalIndexTestBase { static Stream argumentsForBasicReads() { @@ -88,27 +92,20 @@ static Stream argumentsForIndexReads() { } static Stream argumentsForIndexReadsAfterDeletes() { - final Random random = new Random(System.currentTimeMillis()); - return Stream.concat( - SimpleMultidimensionalIndexTest.argumentsForIndexReadsAfterDeletes(), - Stream.of( - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_SLOT.toString(), false, false), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_SLOT.toString(), false, true), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_SLOT.toString(), true, false), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_SLOT.toString(), true, true), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_NODE.toString(), true, true), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), false, false), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), false, true), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), true, false), - Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), true, true) - )); + Stream fixedArgs = Stream.of( + Arguments.of(0x5ca1b13, 1000, 459, BY_NODE.toString(), true, false), + Arguments.of(0xd133451045L, 5000, 1259, BY_NODE.toString(), true, false) + ); + Stream randomArgs = RandomizedTestUtils.randomArguments(random -> { + long seed = random.nextLong(); + int size = random.nextInt(4000) + 1000; + int deletes = random.nextInt(size); + String storageAdapter = random.nextBoolean() ? BY_SLOT.toString() : BY_NODE.toString(); + boolean useHilbertValue = random.nextBoolean(); + boolean useNodeSlotIndex = random.nextBoolean(); + return Arguments.of(seed, size, deletes, storageAdapter, useHilbertValue, useNodeSlotIndex); + }); + return Stream.concat(fixedArgs, randomArgs); } static Stream argumentsForIndexReadsWithDuplicates() { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java index 42f2a06614..711c9495c3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java @@ -91,6 +91,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; @@ -150,6 +152,7 @@ * Tests for {@code VERSION} type indexes. */ @Tag(Tags.RequiresFDB) +@Execution(ExecutionMode.CONCURRENT) public class VersionIndexTest { private static final byte VERSIONSTAMP_CODE = Tuple.from(Versionstamp.complete(new byte[10])).pack()[0]; @RegisterExtension diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index c03b44329d..6100c5aee9 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -71,6 +71,8 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -145,6 +147,7 @@ * Tests related to planning a query with an OR clause into a union plan. */ @Tag(Tags.RequiresFDB) +@Execution(ExecutionMode.CONCURRENT) class FDBOrQueryToUnionTest extends FDBRecordStoreQueryTestBase { private static Stream booleanArgs() { return Stream.of(false, true); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneIndexFailureTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneIndexFailureTest.java index cf20e7ade8..f5847d472e 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneIndexFailureTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneIndexFailureTest.java @@ -578,8 +578,7 @@ private void explicitMergeIndex(Index index, RecordLayerPropertyStorage contextP injectedFailures.removeFailure(LUCENE_GET_FDB_LUCENE_FILE_REFERENCE_ASYNC); } - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setTimer(timer) .build()) { diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintenanceTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintenanceTest.java index 93a12cac99..9cc2fef238 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintenanceTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintenanceTest.java @@ -41,6 +41,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreConcurrentTestBase; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBTransactionPriority; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter; import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer; @@ -60,6 +61,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -112,6 +115,7 @@ * Tests of the consistency of the Lucene Index. */ @Tag(Tags.RequiresFDB) +@Execution(ExecutionMode.CONCURRENT) public class LuceneIndexMaintenanceTest extends FDBRecordStoreConcurrentTestBase { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneIndexMaintenanceTest.class); @@ -1140,6 +1144,7 @@ private void explicitMergeIndex(Index index, FDBRecordStore recordStore = Objects.requireNonNull(schemaSetup.apply(context)); try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() .setRecordStore(recordStore) + .setPriority(FDBTransactionPriority.DEFAULT) .setIndex(index) .setTimer(new FDBStoreTimer()) .build()) { diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java index aa7afdd0a3..bc28ede997 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java @@ -102,6 +102,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -227,6 +229,7 @@ */ @SuppressWarnings({"resource", "SameParameterValue"}) @Tag(Tags.RequiresFDB) +@Execution(ExecutionMode.CONCURRENT) public class LuceneIndexTest extends FDBLuceneTestBase { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneIndexTest.class); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java index d1fbb6b933..5ed647a461 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java @@ -66,6 +66,8 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; @@ -112,6 +114,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +@Execution(ExecutionMode.CONCURRENT) class LuceneOnlineIndexingTest extends FDBRecordStoreTestBase { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneOnlineIndexingTest.class); @@ -151,10 +154,7 @@ void luceneOnlineIndexingTestSimple() throws IOException { } try (final FDBRecordContext context = openContext()) { rebuildIndexMetaData(context, SIMPLE_DOC, index); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) - .setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(index.getName())) { assertTrue(recordStore.isIndexDisabled(index)); indexBuilder.buildIndex(true); } @@ -205,8 +205,7 @@ void luceneOnlineIndexingTestWithRecordUpdates() throws IOException { try (final FDBRecordContext context = openContext(contextProps)) { rebuildIndexMetaData(context, COMPLEX_DOC, index); final RuntimeException stopBuildException = new RuntimeException("stop build"); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setInitialLimit(1) .setConfigLoader(config -> { @@ -264,10 +263,7 @@ void luceneOnlineIndexingTestWithRecordUpdates() throws IOException { // finish the indexing process try (final FDBRecordContext context = openContext(contextProps)) { rebuildIndexMetaData(context, COMPLEX_DOC, index); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) - .setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(index.getName())) { assertTrue(recordStore.isIndexWriteOnly(index)); indexBuilder.buildIndex(true); } @@ -361,8 +357,7 @@ void luceneOnlineIndexingTestWithAllRecordUpdates(long seed) { AtomicInteger counter = new AtomicInteger(0); try (final FDBRecordContext context = openContext(contextProps)) { rebuildIndexMetaData(context, COMPLEX_DOC, index); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setInitialLimit(1) .setLimit(1) @@ -518,8 +513,7 @@ void luceneOnlineIndexingTestAny(Index index, String document, int numRecords, i // build the index .. try (final FDBRecordContext context = openContext()) { rebuildIndexMetaData(context, document, index); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setLimit(transactionLimit) .setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder() @@ -608,8 +602,7 @@ void luceneOnlineIndexingTestMulti() throws IOException { // build the index .. try (final FDBRecordContext context = openContext()) { recordStore = LuceneIndexTestUtils.openRecordStore(context, path, hook); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setTargetIndexes(indexes) .setLimit(transactionLimit) .build()) { @@ -689,8 +682,7 @@ void luceneOnlineIndexingTestGroupingKeys(int groupingCount) { // build the index .. try (final FDBRecordContext context = openContext()) { openRecordStore(context, hook); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .build()) { assertTrue(recordStore.isIndexDisabled(index)); @@ -746,10 +738,7 @@ void luceneOnlineIndexingTestGroupingKeysBackgroundMerge(int groupingCount) thro String[] allFiles = listFiles(index, tuple, groupingCount); int oldLength = allFiles.length; - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) - .setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(index.getName())) { indexBuilder.mergeIndex(); } @@ -831,10 +820,7 @@ void testRecordUpdateBackgroundMerge() throws IOException { String[] allFiles = listFiles(index); int oldLength = allFiles.length; - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) - .setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(index.getName())) { indexBuilder.mergeIndex(); } @@ -878,10 +864,7 @@ void testRecordUpdateBackgroundMerge2() throws IOException { String[] allFiles = listFiles(index); int oldLength = allFiles.length; - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) - .setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(index.getName())) { indexBuilder.mergeIndex(); } @@ -928,10 +911,7 @@ void testRecordUpdateReducedMerge() throws IOException { int loopCounter = 0; for (boolean allDone = false; !allDone; loopCounter++) { int oldLength = listFiles(index).length; - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) - .setIndex(index) - .build()) { + try (OnlineIndexer indexBuilder = newIndexer(index.getName())) { indexBuilder.mergeIndex(); } int newLength = listFiles(index).length; @@ -955,8 +935,7 @@ void luceneOnlineIndexingTestMergesLimit(int mergesLimit) throws IOException { int loopCounter = 0; for (boolean allDone = false; !allDone; loopCounter++) { int oldLength = listFiles(index).length; - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder() .setInitialMergesCountLimit(mergesLimit) @@ -1044,8 +1023,7 @@ void luceneOnlineIndexingTestMerger() { boolean needMerge = populateDataSplitSegments(index, 4, 1); assertTrue(needMerge); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setMaxAttempts(1) .build()) { @@ -1110,8 +1088,7 @@ void luceneOnlineIndexingTestTerrible2Merger() { boolean needMerge = populateDataSplitSegments(index, 4, 1); assertTrue(needMerge); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setMaxAttempts(1) .build()) { @@ -1143,8 +1120,7 @@ void testRecordUpdateReducedMergeForcingAgileSizeQuota(boolean disableAgilityCon FDBStoreTimer timer = new FDBStoreTimer(); try (FDBRecordContext context = openContext(insertProps)) { openRecordStore(context, hook); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setTimer(timer) .build()) { @@ -1191,8 +1167,7 @@ void testRecordUpdateReducedMergeForcingAgileTimeQuota() throws IOException { FDBStoreTimer timer = new FDBStoreTimer(); try (FDBRecordContext context = openContext(insertProps)) { openRecordStore(context, hook); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setTimer(timer) .build()) { @@ -1266,8 +1241,7 @@ void luceneOnlineIndexingTestTerribleRebalance() { boolean needMerge = populateDataSplitSegments(index, 4, 1); assertTrue(needMerge); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setMaxAttempts(1) .build()) { @@ -1342,8 +1316,7 @@ void luceneOnlineIndexingTestTerribleRebalance2ndChance() { boolean needMerge = populateDataSplitSegments(index, 4, 1); assertTrue(needMerge); - try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder() - .setRecordStore(recordStore) + try (OnlineIndexer indexBuilder = newIndexerBuilder() .setIndex(index) .setMaxAttempts(1) .build()) { diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/AgilityContextTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/AgilityContextTest.java index b3051f7e4d..2714a0c5e3 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/AgilityContextTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/AgilityContextTest.java @@ -30,8 +30,9 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContextConfig; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase; import com.apple.foundationdb.record.provider.foundationdb.FDBTransactionPriority; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath; import com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage; -import com.apple.foundationdb.record.util.RandomUtil; +import com.apple.foundationdb.record.test.TestKeySpace; import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.Tuple; import com.apple.foundationdb.tuple.TupleHelpers; @@ -51,7 +52,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -71,7 +71,6 @@ class AgilityContextTest extends FDBRecordStoreTestBase { int loopCount = 20; int threadCount = 5; // if exceeds a certain size, may cause an execution pool deadlock - final String prefix = RandomUtil.randomByteString(ThreadLocalRandom.current(), 100).toString(); private AgilityContext getAgilityContextAgileProp(FDBRecordContext callerContext) { final long timeQuotaMillis = @@ -396,16 +395,18 @@ void napTime(int napTimeMilliseconds) { } } - @ParameterizedTest + @ParameterizedTest(name = "testAgilityContextAtomicAttribute[useAgile={0}]") @BooleanSource void testAgilityContextAtomicAttribute(boolean useAgile) { // assert that commits doesn't happen in he middle of an accept or apply call + final KeySpacePath rawDataPath = pathManager.createPath(TestKeySpace.RAW_DATA); for (int sizeQuota : new int[] {1, 21, 100, 10000}) { final RecordLayerPropertyStorage.Builder insertProps = RecordLayerPropertyStorage.newBuilder() .addProp(LuceneRecordContextProperties.LUCENE_AGILE_COMMIT_SIZE_QUOTA, sizeQuota); IntStream.rangeClosed(0, threadCount).parallel().forEach(threadNum -> { try (FDBRecordContext context = openContext(insertProps)) { + byte[] prefix = rawDataPath.toTuple(context).pack(); final AgilityContext agilityContext = getAgilityContext(context, useAgile); for (int i = 1700; i < 1900; i += 17) { final long iFinal = i; @@ -477,14 +478,15 @@ void testAgilityContextRecoveryPath() { @Test void testAgilityContextRecoveryPath2() { + final KeySpacePath rawDataPath = pathManager.createPath(TestKeySpace.RAW_DATA); final Tuple value = Tuple.from(800, "Green eggs and ham", 0); - final Tuple successTuple = Tuple.from(prefix, "yes"); - final Tuple abortedTuple = Tuple.from(prefix, "abort"); - final Tuple failTuple = Tuple.from(prefix, "no"); + final Tuple successTuple = Tuple.from("yes"); + final Tuple abortedTuple = Tuple.from("abort"); + final Tuple failTuple = Tuple.from("no"); byte[] packedValue = value.pack(); try (FDBRecordContext context = openContext()) { + final Subspace subspace = rawDataPath.toSubspace(context); final AgilityContext agilityContext = getAgilityContext(context, true); - final Subspace subspace = path.toSubspace(context); byte[] keyAborted = subspace.pack(abortedTuple); byte[] keySucceeds = subspace.pack(successTuple); byte[] keyFails = subspace.pack(failTuple); @@ -511,7 +513,7 @@ void testAgilityContextRecoveryPath2() { context.commit(); } try (FDBRecordContext context = openContext()) { - final Subspace subspace = path.toSubspace(context); + final Subspace subspace = rawDataPath.toSubspace(context); byte[] keyAborted = subspace.pack(abortedTuple); byte[] keySucceeds = subspace.pack(successTuple); byte[] keyFails = subspace.pack(failTuple); @@ -525,6 +527,7 @@ void testAgilityContextRecoveryPath2() { @BooleanSource void testAgilityContextAtomicAttributeMultiContext(boolean useAgile) { // assert that commits doesn't happen in he middle of an accept or apply call + final KeySpacePath rawDataPath = pathManager.createPath(TestKeySpace.RAW_DATA); IntStream.rangeClosed(0, threadCount).parallel().forEach(threadNum -> { for (int i = 1700; i < 1900; i += 17) { final long iFinal = i; @@ -532,11 +535,12 @@ void testAgilityContextAtomicAttributeMultiContext(boolean useAgile) { int numWriters = 1; // in this test, two writes may conflict each other if (threadNum < numWriters) { try (FDBRecordContext context = openContext()) { + final Tuple rawDataTuple = rawDataPath.toTuple(context); final AgilityContext agilityContext = getAgilityContext(context, useAgile); agilityContext.accept(aContext -> { for (int j = 0; j < 5; j++) { final Transaction tr = aContext.ensureActive(); - tr.set(Tuple.from(prefix, iFinal, j).pack(), + tr.set(rawDataTuple.add(iFinal).add(j).pack(), Tuple.from(iFinal).pack()); napTime(1); } @@ -549,10 +553,11 @@ void testAgilityContextAtomicAttributeMultiContext(boolean useAgile) { try (FDBRecordContext context = openContext()) { final AgilityContext agilityContext = getAgilityContext(context, useAgile); agilityContext.accept(aContext -> { + final Tuple rawDataTuple = rawDataPath.toTuple(context); long[] values = new long[5]; final Transaction tr = aContext.ensureActive(); for (int j = 4; j >= 0; j--) { - byte[] val = tr.get(Tuple.from(prefix, iFinal, j).pack()).join(); + byte[] val = tr.get(rawDataTuple.add(iFinal).add(j).pack()).join(); values[j] = val == null ? 0 : Tuple.fromBytes(val).getLong(0); } for (int j = 1; j < 5; j++) { @@ -570,7 +575,7 @@ void testAgilityContextAtomicAttributeMultiContext(boolean useAgile) { void testCloseOnCommitFailure() { final byte[] key; try (FDBRecordContext context = openContext()) { - key = this.path.toSubspace(context).pack(Tuple.from(prefix, "a").pack()); + key = this.path.toSubspace(context).pack(Tuple.from("a").pack()); context.ensureActive().set(key, Tuple.from(1).pack()); context.commit(); } @@ -601,7 +606,7 @@ void testCloseOnCommitFailure() { void testAutoCommitVersionStampOuterSleep() throws InterruptedException { final byte[] key; try (FDBRecordContext userContext = openContext()) { - key = this.path.toSubspace(userContext).pack(Tuple.from(prefix, "a").pack()); + key = this.path.toSubspace(userContext).pack(Tuple.from("a").pack()); final AgilityContext agilityContext = AgilityContext.agile(userContext, 2, 10000); AtomicReference firstOperation = new AtomicReference<>(); agilityContext.accept(context -> { @@ -626,7 +631,7 @@ void testAutoCommitVersionStampOuterSleep() throws InterruptedException { void testAutoCommitVersionStampOuterSleepUseApply() throws InterruptedException { final byte[] key; try (FDBRecordContext userContext = openContext()) { - key = this.path.toSubspace(userContext).pack(Tuple.from(prefix, "a").pack()); + key = this.path.toSubspace(userContext).pack(Tuple.from("a").pack()); final AgilityContext agilityContext = AgilityContext.agile(userContext, 2, 10000); AtomicReference firstOperation = new AtomicReference<>(); agilityContext.apply(context -> context.ensureActive() @@ -655,7 +660,7 @@ void testAutoCommitVersionStampOuterSleepUseApply() throws InterruptedException void testAutoCommitVersionStampInnerSleep() { final byte[] key; try (FDBRecordContext userContext = openContext()) { - key = this.path.toSubspace(userContext).pack(Tuple.from(prefix, "a").pack()); + key = this.path.toSubspace(userContext).pack(Tuple.from("a").pack()); final AgilityContext agilityContext = AgilityContext.agile(userContext, 2, 10000); AtomicReference firstOperation = new AtomicReference<>(); agilityContext.accept(context -> { @@ -682,7 +687,7 @@ void testAutoCommitVersionStampInnerSleep() { void testAutoCommitVersionStampInnerSleepUseApply() { final byte[] key; try (FDBRecordContext userContext = openContext()) { - key = this.path.toSubspace(userContext).pack(Tuple.from(prefix, "a").pack()); + key = this.path.toSubspace(userContext).pack(Tuple.from("a").pack()); final AgilityContext agilityContext = AgilityContext.agile(userContext, 2, 10000); AtomicReference firstOperation = new AtomicReference<>(); agilityContext.apply(context -> context.ensureActive() diff --git a/gradle/testing.gradle b/gradle/testing.gradle index f82fb78b13..115163ecbb 100644 --- a/gradle/testing.gradle +++ b/gradle/testing.gradle @@ -223,9 +223,11 @@ tasks.withType(Test).configureEach { task -> task.testFramework.options.excludeTags.add('Performance') if (task.name == 'test') { - task.systemProperty('junit.jupiter.execution.parallel.enabled', 'false') + task.systemProperty('junit.jupiter.execution.parallel.enabled', 'true') task.systemProperty('junit.jupiter.execution.parallel.mode.default', 'same_thread') task.systemProperty('junit.jupiter.execution.parallel.mode.classes.default', 'concurrent') + task.systemProperty('junit.jupiter.execution.parallel.config.strategy', 'fixed') + task.systemProperty('junit.jupiter.execution.parallel.config.fixed.parallelism', '5') } } From 9020db4102a7ecd51c3b858358eb3fe4bd8c9374 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Thu, 19 Dec 2024 11:23:07 -0500 Subject: [PATCH 2/7] Test for running Yaml tests against external server (REL-830) In addition to adding the new class, it reworks the class hierarchy so the tests are in a `YamlIntegrationTests` class, that the other classes extend from, and `JDBCYamlIntegrationTests` which disables the tests that use features unsupported by JDBC. --- yaml-tests/src/test/java/JDBCExternalYamlIntegrationTests.java | 1 - yaml-tests/src/test/java/JDBCInProcessYamlIntegrationTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/yaml-tests/src/test/java/JDBCExternalYamlIntegrationTests.java b/yaml-tests/src/test/java/JDBCExternalYamlIntegrationTests.java index 73145405e6..2888ca373d 100644 --- a/yaml-tests/src/test/java/JDBCExternalYamlIntegrationTests.java +++ b/yaml-tests/src/test/java/JDBCExternalYamlIntegrationTests.java @@ -20,7 +20,6 @@ import com.apple.foundationdb.relational.api.RelationalConnection; import com.apple.foundationdb.relational.yamltests.YamlRunner; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Disabled; diff --git a/yaml-tests/src/test/java/JDBCInProcessYamlIntegrationTests.java b/yaml-tests/src/test/java/JDBCInProcessYamlIntegrationTests.java index 38e5457a09..23b34f4c4e 100644 --- a/yaml-tests/src/test/java/JDBCInProcessYamlIntegrationTests.java +++ b/yaml-tests/src/test/java/JDBCInProcessYamlIntegrationTests.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.relational.jdbc.JDBCURI; import com.apple.foundationdb.relational.server.InProcessRelationalServer; import com.apple.foundationdb.relational.yamltests.YamlRunner; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.AfterAll; From 7d1ea25cdab93415ab4045b6bd241785ed8b8e5f Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 21 Jan 2025 17:57:53 +0000 Subject: [PATCH 3/7] more test fixes to handle deadline exceeded etc --- .../foundationdb/async/RangeSetTest.java | 2 +- .../apple/test/LimitConcurrencyExtension.java | 6 +- .../record/cursors/ConcatCursorTest.java | 3 + .../provider/common/MappedPoolTest.java | 26 ++++--- .../BlockingInAsyncDetectionTest.java | 2 + .../foundationdb/FDBDatabaseRunnerTest.java | 3 + .../foundationdb/FDBDatabaseTest.java | 7 ++ .../foundationdb/FDBMetaDataStoreTest.java | 3 + .../foundationdb/FDBRecordContextTest.java | 6 ++ .../FDBRecordStoreConcurrentTestBase.java | 3 + .../foundationdb/FDBRecordStoreIndexTest.java | 2 +- .../FDBReverseDirectoryCacheTest.java | 3 + .../foundationdb/FDBStoreTimerTest.java | 3 + .../foundationdb/FDBTypedRecordStoreTest.java | 3 + .../foundationdb/KeyValueCursorTest.java | 3 + .../provider/foundationdb/OneOfTest.java | 3 + .../foundationdb/OnlineIndexerMergeTest.java | 2 + .../foundationdb/OnlineIndexerMutualTest.java | 2 + .../foundationdb/OnlineIndexerSimpleTest.java | 5 ++ .../foundationdb/OnlineIndexerTest.java | 3 + .../foundationdb/SynchronizedSessionTest.java | 67 ++++++++++++------- .../indexes/VersionIndexTest.java | 3 + .../keyspace/ResolverCreateHooksTest.java | 3 + .../keyspace/ResolverMappingDigestTest.java | 3 + .../ResolverMappingReplicatorTest.java | 3 + .../HighContentionAllocatorTest.java | 3 + .../interning/StringInterningLayerTest.java | 3 + .../leaderboard/LeaderboardIndexTest.java | 3 + .../runners/TransactionalRunnerTest.java | 3 + .../AbstractSyntheticRecordPlannerTest.java | 3 + .../directory/FDBDirectoryBaseTest.java | 3 + .../directory/FDBDirectoryLockTest.java | 3 + gradle.properties | 1 + gradle/testing.gradle | 5 +- 34 files changed, 158 insertions(+), 38 deletions(-) diff --git a/fdb-extensions/src/test/java/com/apple/foundationdb/async/RangeSetTest.java b/fdb-extensions/src/test/java/com/apple/foundationdb/async/RangeSetTest.java index b830405c33..732f8861ee 100644 --- a/fdb-extensions/src/test/java/com/apple/foundationdb/async/RangeSetTest.java +++ b/fdb-extensions/src/test/java/com/apple/foundationdb/async/RangeSetTest.java @@ -65,7 +65,7 @@ public class RangeSetTest { @RegisterExtension static final TestDatabaseExtension dbExtension = new TestDatabaseExtension(); @RegisterExtension - TestSubspaceExtension rsSubspaceExtension = new TestSubspaceExtension(dbExtension); + final TestSubspaceExtension rsSubspaceExtension = new TestSubspaceExtension(dbExtension); private Database db; private Subspace rsSubspace; private RangeSet rs; diff --git a/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java b/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java index 53245ba67b..794bd568f8 100644 --- a/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java +++ b/fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java @@ -21,16 +21,20 @@ package com.apple.test; import com.google.auto.service.AutoService; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.Extension; import org.junit.jupiter.api.extension.ExtensionContext; +import javax.annotation.Nonnull; import java.util.concurrent.Semaphore; @AutoService(Extension.class) +@Order(0) public class LimitConcurrencyExtension implements BeforeEachCallback, AfterEachCallback { - static Semaphore testConcurrency = new Semaphore(Integer.parseInt(System.getProperty("tests.concurrencyLimit", "10"))); + @Nonnull + static Semaphore testConcurrency = new Semaphore(Integer.parseInt(System.getProperty("com.apple.foundationdb.test.concurrencyLimit", "50"))); @Override public void beforeEach(final ExtensionContext context) throws InterruptedException { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/cursors/ConcatCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/cursors/ConcatCursorTest.java index 3938c0ea33..12184f87a3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/cursors/ConcatCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/cursors/ConcatCursorTest.java @@ -40,6 +40,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -60,8 +61,10 @@ public class ConcatCursorTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); FDBDatabase fdb; FDBRecordContext context; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/common/MappedPoolTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/common/MappedPoolTest.java index 3bc7532e98..a05baacfc4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/common/MappedPoolTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/common/MappedPoolTest.java @@ -21,6 +21,8 @@ package com.apple.foundationdb.record.provider.common; import org.junit.jupiter.api.Test; + +import javax.annotation.Nonnull; import javax.crypto.Cipher; import java.security.GeneralSecurityException; @@ -40,37 +42,43 @@ public class MappedPoolTest { public static String CIPHER = "DES/ECB/PKCS5Padding"; - public static MappedPool MAPPED_POOL = new MappedPool<>(Cipher::getInstance); + + @Nonnull + private MappedPool getPool() { + return new MappedPool<>(Cipher::getInstance); + } @Test public void testCipherPool() throws Exception { + final var mappedPool = getPool(); Cipher lastCipher = null; for (int i = 0; i < 100; i++) { - Cipher cipher = MAPPED_POOL.poll(CIPHER); + Cipher cipher = mappedPool.poll(CIPHER); if (lastCipher != null) { assertSame(cipher, lastCipher); lastCipher = cipher; } assertNotNull(cipher); - assertTrue(MAPPED_POOL.offer(CIPHER, cipher)); + assertTrue(mappedPool.offer(CIPHER, cipher)); } - assertEquals(1, MAPPED_POOL.getPoolSize(CIPHER)); - assertThat(MAPPED_POOL.getKeys(), hasItem(CIPHER)); + assertEquals(1, mappedPool.getPoolSize(CIPHER)); + assertThat(mappedPool.getKeys(), hasItem(CIPHER)); } @Test public void testMaxPoolSize() throws Exception { + final var mappedPool = getPool(); Cipher[] ciphers = new Cipher[1000]; for (int i = 0; i < 1000; i++) { - ciphers[i] = MAPPED_POOL.poll(CIPHER); + ciphers[i] = mappedPool.poll(CIPHER); } for (int i = 0; i < MappedPool.DEFAULT_POOL_SIZE; i++) { - assertTrue(MAPPED_POOL.offer(CIPHER, ciphers[i])); + assertTrue(mappedPool.offer(CIPHER, ciphers[i])); } for (int i = MappedPool.DEFAULT_POOL_SIZE; i < 1000; i++) { - assertFalse(MAPPED_POOL.offer(CIPHER, ciphers[i])); + assertFalse(mappedPool.offer(CIPHER, ciphers[i])); } - assertEquals(64, MAPPED_POOL.getPoolSize(CIPHER)); + assertEquals(64, mappedPool.getPoolSize(CIPHER)); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java index deb86e7aa3..c2d74606fa 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.parallel.Isolated; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -36,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +@Isolated @Tag(Tags.RequiresFDB) class BlockingInAsyncDetectionTest { @RegisterExtension diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseRunnerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseRunnerTest.java index 1a84fa5aaf..c79a5af10b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseRunnerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseRunnerTest.java @@ -38,6 +38,7 @@ import com.google.protobuf.Message; import org.apache.logging.log4j.ThreadContext; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -70,8 +71,10 @@ @Tag(Tags.RequiresFDB) public class FDBDatabaseRunnerTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); FDBDatabase database; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java index 6037031f56..a9653aaaeb 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java @@ -40,12 +40,15 @@ import com.apple.test.BooleanSource; import com.apple.test.Tags; import com.google.protobuf.Message; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.api.parallel.Resources; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.slf4j.Logger; @@ -83,8 +86,10 @@ class FDBDatabaseTest { @Nonnull private static final Logger LOGGER = LoggerFactory.getLogger(FDBDatabaseTest.class); @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); @Test @@ -216,6 +221,7 @@ void cachedReadVersionWithRetryLoops(boolean async) throws InterruptedException, @ParameterizedTest(name = "testJoinNowOnCompletedFuture (behavior = {0})") @EnumSource(BlockingInAsyncDetection.class) + @ResourceLock(Resources.GLOBAL) void testJoinNowOnCompletedFuture(BlockingInAsyncDetection behavior) { FDBDatabaseFactory factory = dbExtension.getDatabaseFactory(); factory.setBlockingInAsyncDetection(behavior); @@ -231,6 +237,7 @@ void testJoinNowOnCompletedFuture(BlockingInAsyncDetection behavior) { @ParameterizedTest(name = "testJoinNowOnNonCompletedFuture (behavior = {0})") @EnumSource(BlockingInAsyncDetection.class) + @ResourceLock(Resources.GLOBAL) void testJoinNowOnNonCompletedFuture(BlockingInAsyncDetection behavior) { FDBDatabaseFactory factory = dbExtension.getDatabaseFactory(); factory.setBlockingInAsyncDetection(behavior); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java index 9c3e7c65af..05a0245a72 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java @@ -66,6 +66,7 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.ExtensionRegistry; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -103,8 +104,10 @@ @Tag(Tags.RequiresFDB) public class FDBMetaDataStoreTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); FDBDatabase fdb; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java index f517382b6a..05e7ee5498 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java @@ -43,9 +43,12 @@ import com.google.common.base.Utf8; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.api.parallel.Resources; import org.junit.jupiter.params.ParameterizedTest; import javax.annotation.Nonnull; @@ -82,8 +85,10 @@ @Tag(Tags.RequiresFDB) public class FDBRecordContextTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); // A list of transaction IDs where the left item is the original ID and the right item is the expected @@ -333,6 +338,7 @@ public void getReadVersionWithWeakReadSemantics() { } } + @ResourceLock(Resources.GLOBAL) @Test public void getReadVersionAtBatch() { final FDBStoreTimer timer = new FDBStoreTimer(); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java index d8e3a8b651..f6ea4c0b19 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java @@ -39,6 +39,7 @@ import com.apple.test.Tags; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; @@ -57,8 +58,10 @@ @Tag(Tags.RequiresFDB) public class FDBRecordStoreConcurrentTestBase { @RegisterExtension + @Order(0) protected final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) protected final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private static final Logger LOGGER = LoggerFactory.getLogger(FDBRecordStoreConcurrentTestBase.class); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java index 7f8738a07a..295279561d 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java @@ -2250,7 +2250,7 @@ public IndexState needRebuildIndex(Index index, long recordCount, boolean indexO recordStore.clearAndMarkIndexWriteOnly("index-1").join(); context.commit(); } - try (OnlineIndexer onlineIndexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "index-1")) { + try (OnlineIndexer onlineIndexBuilder = newIndexerBuilder().setIndex("index-1").build()) { onlineIndexBuilder.buildIndex(); } try (FDBRecordContext context = openContext()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBReverseDirectoryCacheTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBReverseDirectoryCacheTest.java index 6aaafbeacb..a679d2d3f4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBReverseDirectoryCacheTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBReverseDirectoryCacheTest.java @@ -40,6 +40,7 @@ import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -80,8 +81,10 @@ @Tag(Tags.RequiresFDB) public class FDBReverseDirectoryCacheTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private static final Logger logger = LoggerFactory.getLogger(FDBReverseDirectoryCacheTest.class); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimerTest.java index 33e4af6238..335a6833f0 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimerTest.java @@ -43,6 +43,7 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -76,8 +77,10 @@ @Execution(ExecutionMode.CONCURRENT) public class FDBStoreTimerTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); FDBDatabase fdb; KeySpacePath path; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java index b6201caf59..41797f2f49 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.tuple.Tuple; import com.apple.test.Tags; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -47,8 +48,10 @@ @Tag(Tags.RequiresFDB) public class FDBTypedRecordStoreTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); FDBDatabase fdb; KeySpacePath path; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java index d97e4b0ef9..9606e971b9 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java @@ -41,6 +41,7 @@ import com.apple.foundationdb.tuple.Tuple; import com.apple.test.Tags; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -64,8 +65,10 @@ @Tag(Tags.RequiresFDB) public class KeyValueCursorTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase fdb; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OneOfTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OneOfTest.java index cbad474586..3ab54f3a47 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OneOfTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OneOfTest.java @@ -34,6 +34,7 @@ import com.apple.test.Tags; import com.google.protobuf.Message; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -47,8 +48,10 @@ @Tag(Tags.RequiresFDB) public class OneOfTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase fdb; private KeySpacePath path; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java index 37b42aa636..50ff413d52 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java @@ -50,6 +50,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -72,6 +73,7 @@ * Tests specifically of {@link OnlineIndexer#mergeIndex()}. */ @Tag(Tags.RequiresFDB) +@Isolated public class OnlineIndexerMergeTest extends FDBRecordStoreConcurrentTestBase { private static final String INDEX_NAME = "mergableIndex"; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMutualTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMutualTest.java index 7258a25577..fc5097098b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMutualTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMutualTest.java @@ -41,6 +41,7 @@ import com.apple.test.Tags; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.slf4j.Logger; @@ -75,6 +76,7 @@ * Tests for mutually building indexes {@link OnlineIndexer}. */ @Tag(Tags.Slow) +@Isolated class OnlineIndexerMutualTest extends OnlineIndexerTest { private static final Logger LOGGER = LoggerFactory.getLogger(OnlineIndexerMutualTest.class); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerSimpleTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerSimpleTest.java index 803c323a45..715537b298 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerSimpleTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerSimpleTest.java @@ -45,6 +45,8 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.api.parallel.Resources; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -382,6 +384,7 @@ public void readableAtEnd() { } } + @ResourceLock(Resources.GLOBAL) @Test public void logsEnd() { List records = LongStream.range(0, 50).mapToObj(val -> @@ -846,6 +849,7 @@ public void testTimeLimit() { } } + @ResourceLock(Resources.GLOBAL) @Test public void testLogInterval() { final int limit = 20; @@ -901,6 +905,7 @@ public void testLogInterval() { .setTimer(timer) .setProgressLogIntervalMillis(0) .setLimit(limit) + .setPriority(FDBTransactionPriority.BATCH) .setConfigLoader(old -> { // Ensure that time limit is exceeded try { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java index 6f9bf4cda3..546399e12b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java @@ -41,6 +41,7 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.extension.RegisterExtension; @@ -62,8 +63,10 @@ @Tag(Tags.RequiresFDB) public abstract class OnlineIndexerTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); FDBDatabase fdb; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java index ee42087d61..4c6cfb193b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.provider.foundationdb; import com.apple.foundationdb.async.AsyncUtil; +import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.provider.foundationdb.synchronizedsession.SynchronizedSessionRunner; import com.apple.foundationdb.record.test.FDBDatabaseExtension; import com.apple.foundationdb.record.test.TestKeySpace; @@ -29,21 +30,27 @@ import com.apple.foundationdb.synchronizedsession.SynchronizedSessionLockedException; import com.apple.test.Tags; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Random; +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.Deque; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests for {@link com.apple.foundationdb.record.provider.foundationdb.synchronizedsession.SynchronizedSessionRunner}. @@ -51,9 +58,13 @@ @Tag(Tags.RequiresFDB) public abstract class SynchronizedSessionTest { private static final Logger LOGGER = LoggerFactory.getLogger(SynchronizedSessionTest.class); + private static final AtomicInteger threadCounter = new AtomicInteger(); + @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase database; @@ -64,8 +75,6 @@ public abstract class SynchronizedSessionTest { private static final long DEFAULT_LEASE_LENGTH_MILLIS = 250; - private Random random = new Random(); - private SynchronizedSessionTest(boolean runAsync) { this.runAsync = runAsync; } @@ -223,18 +232,16 @@ public void takeLaterOneWhenThereAreDifferentLeaseEndTimes() throws Exception { UUID session1Id = session1Runner1.getSessionId(); try (SynchronizedSessionRunner session1Runner2 = database.newRunner().joinSynchronizedSession(lockSubspace1, session1Id, DEFAULT_LEASE_LENGTH_MILLIS * 2)) { - Thread run1 = new Thread(() -> checkActive(session1Runner1)); - Thread run2 = new Thread(() -> checkActive(session1Runner2)); - - AtomicBoolean threadsHaveExceptions = new AtomicBoolean(false); - logExceptionsInThreads(threadsHaveExceptions); + Deque uncaughtExceptions = new ConcurrentLinkedDeque<>(); + Thread run1 = createThread(uncaughtExceptions, () -> checkActive(session1Runner1)); + Thread run2 = createThread(uncaughtExceptions, () -> checkActive(session1Runner2)); run1.start(); run2.start(); run1.join(); run2.join(); - assertTrue(!threadsHaveExceptions.get()); + assertNoExceptions(uncaughtExceptions); // Runner 1 set the lease end time to 2 seconds in the future, while Runner 2 set it to 3 seconds, // the later one should be honoured. So a new session shouldn't take the lock until 3 seconds. @@ -266,7 +273,8 @@ private void testRenewLeaseContinuously(boolean reuseRunner) throws Exception { try (SynchronizedSessionRunner session1Runner0 = database.newRunner().startSynchronizedSession(lockSubspace1, 1_000)) { UUID session1 = session1Runner0.getSessionId(); AtomicBoolean session1Stopped = new AtomicBoolean(false); - Thread longSession = new Thread(() -> { + Deque uncaughtExceptions = new ConcurrentLinkedDeque<>(); + Thread longSession = createThread(uncaughtExceptions, () -> { for (int i = 0; i < 10; i++) { try { Thread.sleep(500); @@ -279,37 +287,27 @@ private void testRenewLeaseContinuously(boolean reuseRunner) throws Exception { } session1Stopped.set(true); }); - Thread tryStartSession = new Thread(() -> { + Thread tryStartSession = createThread(uncaughtExceptions, () -> { while (!session1Stopped.get()) { assertFailedStartSession(lockSubspace1); try { - Thread.sleep(random.nextInt(500)); + Thread.sleep(ThreadLocalRandom.current().nextInt(500)); } catch (InterruptedException e) { e.printStackTrace(); } } }); - AtomicBoolean threadsHaveExceptions = new AtomicBoolean(false); - logExceptionsInThreads(threadsHaveExceptions); - longSession.start(); tryStartSession.start(); tryStartSession.join(); - assertTrue(!threadsHaveExceptions.get()); + assertNoExceptions(uncaughtExceptions); session1Runner0.endSession(); } } - private void logExceptionsInThreads(AtomicBoolean threadsHaveExceptions) { - Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> { - threadsHaveExceptions.set(true); - LOGGER.error("Exception in a thread", thread.toString(), exception); - }); - } - private SynchronizedSessionRunner newRunnerStartSession(Subspace lockSubspace) { return database.newRunner().startSynchronizedSession(lockSubspace, DEFAULT_LEASE_LENGTH_MILLIS); } @@ -345,6 +343,27 @@ private void assertFailedContinueSession(SynchronizedSessionRunner synchronizedS assertEquals("Failed to continue the session", exception.getMessage()); } + @Nonnull + private static Thread createThread(Collection uncaughtExceptions, Runnable task) { + Thread t = new Thread(task); + t.setName("synchronized-session-test-" + threadCounter.getAndIncrement()); + t.setDaemon(true); + t.setUncaughtExceptionHandler((thread, exception) -> { + LOGGER.error(KeyValueLogMessage.of("Uncaught exception in thread", + "thread", thread.getName() + ), exception); + uncaughtExceptions.add(exception); + }); + return t; + } + + private static void assertNoExceptions(Collection uncaughtExceptions) { + if (!uncaughtExceptions.isEmpty()) { + Throwable t = uncaughtExceptions.iterator().next(); + fail("Found " + uncaughtExceptions.size() + " unexpected exceptions", t); + } + } + private void waitLongEnough() throws InterruptedException { Thread.sleep(DEFAULT_LEASE_LENGTH_MILLIS + 100); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java index 711c9495c3..c82eefb403 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexTest.java @@ -88,6 +88,7 @@ import com.google.protobuf.Message; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -156,8 +157,10 @@ public class VersionIndexTest { private static final byte VERSIONSTAMP_CODE = Tuple.from(Versionstamp.complete(new byte[10])).pack()[0]; @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private RecordMetaData metaData; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverCreateHooksTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverCreateHooksTest.java index affc4fa028..c51404caad 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverCreateHooksTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverCreateHooksTest.java @@ -30,6 +30,7 @@ import com.apple.foundationdb.tuple.Tuple; import com.apple.test.Tags; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -45,8 +46,10 @@ @Tag(Tags.RequiresFDB) class ResolverCreateHooksTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase database; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingDigestTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingDigestTest.java index f33bdea1b1..3d0f7e6c55 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingDigestTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingDigestTest.java @@ -32,6 +32,7 @@ import com.apple.test.Tags; import com.google.common.collect.ImmutableSortedMap; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -55,8 +56,10 @@ @Tag(Tags.RequiresFDB) public class ResolverMappingDigestTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase database; private Random random = new Random(); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingReplicatorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingReplicatorTest.java index e893b656ad..d3855ad627 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingReplicatorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolverMappingReplicatorTest.java @@ -35,6 +35,7 @@ import com.apple.test.Tags; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -60,8 +61,10 @@ @Tag(Tags.RequiresFDB) public abstract class ResolverMappingReplicatorTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); protected LocatableResolver primary; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/HighContentionAllocatorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/HighContentionAllocatorTest.java index 4c0065fb95..68f8f888a7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/HighContentionAllocatorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/HighContentionAllocatorTest.java @@ -37,6 +37,7 @@ import com.apple.foundationdb.tuple.Tuple; import com.apple.test.Tags; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -66,8 +67,10 @@ @Tag(Tags.RequiresFDB) class HighContentionAllocatorTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase database; private KeySpacePath path; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/StringInterningLayerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/StringInterningLayerTest.java index 8a8074cbe7..ba28723109 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/StringInterningLayerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/layers/interning/StringInterningLayerTest.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.tuple.Tuple; import com.apple.test.Tags; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -66,8 +67,10 @@ @Tag(Tags.RequiresFDB) class StringInterningLayerTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase database; private Subspace testSubspace; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/leaderboard/LeaderboardIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/leaderboard/LeaderboardIndexTest.java index b93fe02e53..508dffb1bf 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/leaderboard/LeaderboardIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/leaderboard/LeaderboardIndexTest.java @@ -68,6 +68,7 @@ import com.google.protobuf.Message; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -98,8 +99,10 @@ @Tag(Tags.RequiresFDB) public class LeaderboardIndexTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase fdb; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/runners/TransactionalRunnerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/runners/TransactionalRunnerTest.java index 58db04c972..7a28745232 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/runners/TransactionalRunnerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/runners/TransactionalRunnerTest.java @@ -36,6 +36,7 @@ import com.apple.test.Tags; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -80,8 +81,10 @@ @Execution(ExecutionMode.CONCURRENT) class TransactionalRunnerTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase database; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/synthetic/AbstractSyntheticRecordPlannerTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/synthetic/AbstractSyntheticRecordPlannerTest.java index 7b367a7357..841fe8459a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/synthetic/AbstractSyntheticRecordPlannerTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/synthetic/AbstractSyntheticRecordPlannerTest.java @@ -40,6 +40,7 @@ import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.extension.RegisterExtension; @@ -56,8 +57,10 @@ @API(API.Status.EXPERIMENTAL) public abstract class AbstractSyntheticRecordPlannerTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); protected FDBDatabase fdb; diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryBaseTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryBaseTest.java index 592c1fadd9..4ebd353005 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryBaseTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryBaseTest.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.subspace.Subspace; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.extension.RegisterExtension; import javax.annotation.Nonnull; @@ -46,8 +47,10 @@ */ public abstract class FDBDirectoryBaseTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); protected FDBDatabase fdb; protected KeySpacePath path; diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryLockTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryLockTest.java index b06bd245b1..98f25709e4 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryLockTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryLockTest.java @@ -34,6 +34,7 @@ import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -53,8 +54,10 @@ @Tag(Tags.RequiresFDB) class FDBDirectoryLockTest { @RegisterExtension + @Order(0) final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @RegisterExtension + @Order(1) final TestKeySpacePathManagerExtension pathManager = new TestKeySpacePathManagerExtension(dbExtension); private FDBDatabase fdb; diff --git a/gradle.properties b/gradle.properties index c403d0a0a1..a26ba490ba 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,5 +35,6 @@ apiVersion=710 mavenLocalEnabled=false org.gradle.daemon=true +testConcurrency=10 url = 'https://github.com/FoundationDB/fdb-record-layer/' diff --git a/gradle/testing.gradle b/gradle/testing.gradle index 115163ecbb..78f1965493 100644 --- a/gradle/testing.gradle +++ b/gradle/testing.gradle @@ -224,10 +224,11 @@ tasks.withType(Test).configureEach { task -> if (task.name == 'test') { task.systemProperty('junit.jupiter.execution.parallel.enabled', 'true') - task.systemProperty('junit.jupiter.execution.parallel.mode.default', 'same_thread') + task.systemProperty('junit.jupiter.execution.parallel.mode.default', 'concurrent') task.systemProperty('junit.jupiter.execution.parallel.mode.classes.default', 'concurrent') task.systemProperty('junit.jupiter.execution.parallel.config.strategy', 'fixed') - task.systemProperty('junit.jupiter.execution.parallel.config.fixed.parallelism', '5') + task.systemProperty('junit.jupiter.execution.parallel.config.fixed.parallelism', "${testConcurrency}") + task.systemProperty('com.apple.foundationdb.test.concurrencyLimit', "${testConcurrency}") } } From aa4c73b06e150911c91189ea59d1d1592021fd80 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 21 Jan 2025 18:43:40 +0000 Subject: [PATCH 4/7] limit boolean values to a single thread ; exclude parallel testing on relational subprojects --- .../record/provider/foundationdb/SynchronizedSessionTest.java | 2 +- .../record/query/plan/cascades/BooleanValueTest.java | 3 +++ gradle.properties | 2 +- gradle/testing.gradle | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java index 4c6cfb193b..2656ad6140 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/SynchronizedSessionTest.java @@ -347,7 +347,7 @@ private void assertFailedContinueSession(SynchronizedSessionRunner synchronizedS private static Thread createThread(Collection uncaughtExceptions, Runnable task) { Thread t = new Thread(task); t.setName("synchronized-session-test-" + threadCounter.getAndIncrement()); - t.setDaemon(true); + t.setDaemon(false); t.setUncaughtExceptionHandler((thread, exception) -> { LOGGER.error(KeyValueLogMessage.of("Uncaught exception in thread", "thread", thread.getName() diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/BooleanValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/BooleanValueTest.java index 0531fac330..b5a6f62384 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/BooleanValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/BooleanValueTest.java @@ -51,6 +51,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; @@ -71,6 +73,7 @@ *
  • Nullability tests.
    • * */ +@Execution(ExecutionMode.SAME_THREAD) // necessary as the debugger can do weird things with the current quantifier if run in multiple threads class BooleanValueTest { private static final Type.Enum ENUM_TYPE_FOR_TEST = new Type.Enum(false, List.of( diff --git a/gradle.properties b/gradle.properties index a26ba490ba..84550c3d90 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,6 +35,6 @@ apiVersion=710 mavenLocalEnabled=false org.gradle.daemon=true -testConcurrency=10 +testConcurrency=5 url = 'https://github.com/FoundationDB/fdb-record-layer/' diff --git a/gradle/testing.gradle b/gradle/testing.gradle index 78f1965493..6900c24bae 100644 --- a/gradle/testing.gradle +++ b/gradle/testing.gradle @@ -222,7 +222,7 @@ tasks.withType(Test).configureEach { task -> } task.testFramework.options.excludeTags.add('Performance') - if (task.name == 'test') { + if (task.name == 'test' && project.name in ['fdb-extensions', 'fdb-record-layer-core', 'fdb-record-layer-lucene', 'fdb-record-layer-icu', 'fdb-record-layer-spatial']) { task.systemProperty('junit.jupiter.execution.parallel.enabled', 'true') task.systemProperty('junit.jupiter.execution.parallel.mode.default', 'concurrent') task.systemProperty('junit.jupiter.execution.parallel.mode.classes.default', 'concurrent') From 951a9dfee79f4413754c844d7ee3fc1dbdc063e1 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 27 Jan 2025 11:24:29 +0000 Subject: [PATCH 5/7] try at low parallelism to begin with --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 84550c3d90..bac1caf17d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,6 +35,6 @@ apiVersion=710 mavenLocalEnabled=false org.gradle.daemon=true -testConcurrency=5 +testConcurrency=2 url = 'https://github.com/FoundationDB/fdb-record-layer/' From fa992468138ba65aef74372b555213d8d0a7c53a Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 27 Jan 2025 16:21:35 +0000 Subject: [PATCH 6/7] update test concurrency to 5 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index bac1caf17d..84550c3d90 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,6 +35,6 @@ apiVersion=710 mavenLocalEnabled=false org.gradle.daemon=true -testConcurrency=2 +testConcurrency=5 url = 'https://github.com/FoundationDB/fdb-record-layer/' From 91033dc2cc104f49ac81a72c832c8c35fc328285 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 27 Jan 2025 18:12:41 +0000 Subject: [PATCH 7/7] isolate remote fetch tests --- .../record/provider/foundationdb/RemoteFetchTest.java | 2 +- .../record/provider/foundationdb/RemoteFetchTestBase.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTest.java index 152e5c871e..4893a63efa 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.CursorStreamingMode; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.IndexEntry; +import com.apple.foundationdb.record.IndexFetchMethod; import com.apple.foundationdb.record.IndexScanType; import com.apple.foundationdb.record.IsolationLevel; import com.apple.foundationdb.record.RecordCoreStorageException; @@ -34,7 +35,6 @@ import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Query; -import com.apple.foundationdb.record.IndexFetchMethod; import com.apple.foundationdb.record.query.plan.plans.RecordQueryComparatorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.tuple.Tuple; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTestBase.java index 2873244b52..da33d9cb6b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RemoteFetchTestBase.java @@ -40,6 +40,7 @@ import com.apple.test.Tags; import com.google.protobuf.Message; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.parallel.Isolated; import org.junit.jupiter.params.provider.Arguments; import javax.annotation.Nonnull; @@ -59,6 +60,7 @@ * Base class for RemoteFetch tests. */ @Tag(Tags.RequiresFDB) +@Isolated public class RemoteFetchTestBase extends FDBRecordStoreQueryTestBase { protected static final RecordQuery NUM_VALUES_LARGER_THAN_990 = RecordQuery.newBuilder()