diff --git a/compliance/repository/pom.xml b/compliance/repository/pom.xml index b5e75f0c7b3..95e48a074a7 100644 --- a/compliance/repository/pom.xml +++ b/compliance/repository/pom.xml @@ -50,6 +50,18 @@ ${project.version} war + + org.openjdk.jmh + jmh-core + ${jmhVersion} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmhVersion} + test + diff --git a/compliance/repository/src/test/java/org/eclipse/rdf4j/repository/sparql/federation/RepositoryFederatedServiceIntegrationTest.java b/compliance/repository/src/test/java/org/eclipse/rdf4j/repository/sparql/federation/RepositoryFederatedServiceIntegrationTest.java index 763d3c2a6d2..44508c88e9c 100644 --- a/compliance/repository/src/test/java/org/eclipse/rdf4j/repository/sparql/federation/RepositoryFederatedServiceIntegrationTest.java +++ b/compliance/repository/src/test/java/org/eclipse/rdf4j/repository/sparql/federation/RepositoryFederatedServiceIntegrationTest.java @@ -39,6 +39,23 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import com.google.common.collect.Lists; @@ -47,8 +64,17 @@ * * @author Andreas Schwarte */ +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@BenchmarkMode({ Mode.AverageTime }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public class RepositoryFederatedServiceIntegrationTest { + @Param({ "10", "50", "100", "500", "1000", "5000" }) + private int numThreads; + private static final ValueFactory vf = SimpleValueFactory.getInstance(); private SailRepository serviceRepo; @@ -56,6 +82,7 @@ public class RepositoryFederatedServiceIntegrationTest { private RepositoryFederatedService federatedService; @BeforeEach + @Setup(Level.Iteration) public void before() { serviceRepo = new SailRepository(new MemoryStore()); serviceRepo.init(); @@ -74,6 +101,7 @@ public FederatedService getService(String serviceUrl) throws QueryEvaluationExce } @AfterEach + @TearDown((Level.Iteration)) public void after() { federatedService.shutdown(); localRepo.shutDown(); @@ -277,6 +305,7 @@ public void test8a_subSelectAll() { } @Test + @Benchmark public void test9_connectionHandling() throws Exception { /* @@ -297,7 +326,7 @@ public void test9_connectionHandling() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(5); try { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < numThreads; i++) { executor.submit(() -> { String query = "SELECT ?var WHERE { SERVICE { ?s ?p ?var } }"; diff --git a/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/MinimalContextNowTest.java b/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/MinimalContextNowTest.java index a4d5af4ebb5..23cd17560c7 100644 --- a/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/MinimalContextNowTest.java +++ b/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/MinimalContextNowTest.java @@ -16,6 +16,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -27,8 +28,33 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@BenchmarkMode({ Mode.AverageTime }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public class MinimalContextNowTest { + @Param({ "10", "50", "100", "500", "1000", "5000" }) + private int numberOfThreads; + @Test public void testNow() { // Tests that the now value is correctly initialized. @@ -45,9 +71,9 @@ public void testNow() { } @Test + @Benchmark public void testConcurrentAccessToNow() throws ExecutionException, InterruptedException { int numberOfIterations = 100; - int numberOfThreads = 10; ExecutorService executorService = Executors.newCachedThreadPool(); try { diff --git a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/benchmarks/ReadWriteLockManagerBenchmark.java b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/benchmarks/ReadWriteLockManagerBenchmark.java index e9f5542be69..93383f868a7 100644 --- a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/benchmarks/ReadWriteLockManagerBenchmark.java +++ b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/benchmarks/ReadWriteLockManagerBenchmark.java @@ -24,6 +24,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; @@ -41,6 +42,9 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) public class ReadWriteLockManagerBenchmark extends BaseLockManagerBenchmark { + @Param({ "10", "50", "100", "500", "1000", "5000" }) + public int numThreads; + public static void main(String[] args) throws RunnerException { Options opt = new OptionsBuilder().include("ReadWriteLockManagerBenchmark.*") // adapt to run other benchmark // tests @@ -58,7 +62,7 @@ public void onlyReadLocksNoContention(Blackhole blackhole) throws Exception { AbstractReadWriteLockManager lockManager = getReadWriteLockManager(); - threads(100, () -> { + threads(numThreads, () -> { readLocks(lockManager, 100, blackhole); @@ -71,7 +75,7 @@ public void onlyWriteLocks(Blackhole blackhole) throws Exception { AbstractReadWriteLockManager lockManager = getReadWriteLockManager(); - threads(1000, () -> { + threads(numThreads, () -> { try { Lock lock = lockManager.getWriteLock(); lock.release(); @@ -94,7 +98,7 @@ public void mixedReadHeavy(Blackhole blackhole) throws Exception { AbstractReadWriteLockManager lockManager = getReadWriteLockManager(); - threads(100, () -> { + threads(numThreads, () -> { try { Lock lock = lockManager.getWriteLock(); lock.release(); @@ -115,7 +119,7 @@ public void readPriority(Blackhole blackhole) throws Exception { AbstractReadWriteLockManager lockManager = getReadWriteLockManager(); - threads(100, () -> { + threads(numThreads, () -> { try { Lock readLock1 = lockManager.getReadLock(); @@ -190,7 +194,7 @@ public void mixed(Blackhole blackhole) throws Exception { AbstractReadWriteLockManager lockManager = getReadWriteLockManager(); - threads(1000, () -> { + threads(numThreads, () -> { try { Lock lock = lockManager.getWriteLock(); lock.release(); diff --git a/core/sail/api/src/test/java/org/eclipse/rdf4j/sail/helpers/AbstractSailTest.java b/core/sail/api/src/test/java/org/eclipse/rdf4j/sail/helpers/AbstractSailTest.java index c1c64931f40..67b53e5a5c5 100644 --- a/core/sail/api/src/test/java/org/eclipse/rdf4j/sail/helpers/AbstractSailTest.java +++ b/core/sail/api/src/test/java/org/eclipse/rdf4j/sail/helpers/AbstractSailTest.java @@ -31,13 +31,40 @@ * * @author Jeen Broekstra */ +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@BenchmarkMode({ Mode.AverageTime }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public class AbstractSailTest { + @Param({ "10", "50", "100", "500", "1000", "5000" }) + private int count; + AbstractSail subject; private final Random random = new Random(43252333); @BeforeEach + @Setup(Level.Iteration) public void setUp() { subject = new AbstractSail() { @@ -96,8 +123,8 @@ public void testExplicitInitTwice() { } @Test + @Benchmark public void testConcurrentAutoInit() throws Exception { - int count = 200; CountDownLatch latch = new CountDownLatch(count); for (int i = 0; i < count; i++) { diff --git a/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/OverflowBenchmarkConcurrent.java b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/OverflowBenchmarkConcurrent.java index eef34f93d1c..024bb8c84c1 100644 --- a/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/OverflowBenchmarkConcurrent.java +++ b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/OverflowBenchmarkConcurrent.java @@ -54,6 +54,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -78,6 +79,9 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) public class OverflowBenchmarkConcurrent { + @Param({ "5", "25", "50", "250", "500", "2500" }) + public int numThreadsHalf; + @Setup(Level.Trial) public void setup() { ((Logger) (LoggerFactory @@ -123,7 +127,7 @@ public void manyConcurrentTransactions() throws IOException { CountDownLatch countDownLatch = new CountDownLatch(1); - for (int i = 0; i < 38; i++) { + for (int i = 0; i < numThreadsHalf; i++) { var seed = i + 485924; { Future submit = executorService.submit(() -> { diff --git a/core/sail/lucene/pom.xml b/core/sail/lucene/pom.xml index a0e0c7f070f..c9252913b63 100644 --- a/core/sail/lucene/pom.xml +++ b/core/sail/lucene/pom.xml @@ -78,5 +78,17 @@ junit-vintage-engine test + + org.openjdk.jmh + jmh-core + ${jmhVersion} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmhVersion} + test + diff --git a/core/sail/lucene/src/test/java/org/eclipse/rdf4j/sail/lucene/impl/AbstractGenericLuceneTest.java b/core/sail/lucene/src/test/java/org/eclipse/rdf4j/sail/lucene/impl/AbstractGenericLuceneTest.java index 1e09c30169e..67bc5588940 100644 --- a/core/sail/lucene/src/test/java/org/eclipse/rdf4j/sail/lucene/impl/AbstractGenericLuceneTest.java +++ b/core/sail/lucene/src/test/java/org/eclipse/rdf4j/sail/lucene/impl/AbstractGenericLuceneTest.java @@ -67,11 +67,38 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Timeout(value = 10, unit = TimeUnit.MINUTES) +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@BenchmarkMode({ Mode.AverageTime }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public abstract class AbstractGenericLuceneTest { + + @Param({ "10", "50", "100", "500", "1000", "5000" }) + int numThreads; + protected static final ValueFactory vf = SimpleValueFactory.getInstance(); public static final String QUERY_STRING; @@ -120,6 +147,7 @@ public abstract class AbstractGenericLuceneTest { protected abstract void configure(LuceneSail sail) throws IOException; @BeforeEach + @Setup(Level.Iteration) public void setUp() throws Exception { // set logging, uncomment this to get better logging for debugging // org.apache.log4j.BasicConfigurator.configure(); @@ -151,6 +179,7 @@ public void setUp() throws Exception { } @AfterEach + @TearDown(Level.Iteration) public void tearDown() throws RepositoryException { try { if (connection != null) { @@ -767,8 +796,8 @@ public void testPropertyVar() throws MalformedQueryException, RepositoryExceptio } @Test + @Benchmark public void testMultithreadedAdd() throws InterruptedException { - int numThreads = 3; final CountDownLatch startLatch = new CountDownLatch(1); final CountDownLatch endLatch = new CountDownLatch(numThreads); final Set exceptions = ConcurrentHashMap.newKeySet(); diff --git a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ConcurrentQueryBenchmark.java b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ConcurrentQueryBenchmark.java index 1785a1f7352..78c9dcbc740 100644 --- a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ConcurrentQueryBenchmark.java +++ b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ConcurrentQueryBenchmark.java @@ -25,6 +25,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -42,11 +43,14 @@ @State(Scope.Benchmark) @Warmup(iterations = 5) @BenchmarkMode({ Mode.AverageTime }) -@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) @Measurement(iterations = 5) @OutputTimeUnit(TimeUnit.MILLISECONDS) public class ConcurrentQueryBenchmark extends BaseConcurrentBenchmark { + @Param({ "10", "50", "100", "500", "1000", "5000" }) + public int numThreads; + private SailRepository repository; public static void main(String[] args) throws RunnerException { @@ -82,7 +86,7 @@ public void tearDown() throws Exception { @Benchmark public void hasStatement(Blackhole blackhole) throws Exception { - threads(100, () -> { + threads(numThreads, () -> { try (SailRepositoryConnection connection = repository.getConnection()) { for (int i = 0; i < 100; i++) { boolean b = connection.hasStatement(null, null, null, true); @@ -95,7 +99,7 @@ public void hasStatement(Blackhole blackhole) throws Exception { @Benchmark public void hasStatementSharedConnection(Blackhole blackhole) throws Exception { try (SailRepositoryConnection connection = repository.getConnection()) { - threads(100, () -> { + threads(numThreads, () -> { for (int i = 0; i < 100; i++) { boolean b = connection.hasStatement(null, null, null, true); blackhole.consume(b); @@ -106,7 +110,7 @@ public void hasStatementSharedConnection(Blackhole blackhole) throws Exception { @Benchmark public void getNamespaces(Blackhole blackhole) throws Exception { - threads(100, () -> { + threads(numThreads, () -> { try (SailRepositoryConnection connection = repository.getConnection()) { for (int i = 0; i < 100; i++) { blackhole.consume(connection.getNamespaces().stream().count()); @@ -118,7 +122,7 @@ public void getNamespaces(Blackhole blackhole) throws Exception { @Benchmark public void getNamespacesSharedConnection(Blackhole blackhole) throws Exception { try (SailRepositoryConnection connection = repository.getConnection()) { - threads(100, () -> { + threads(numThreads, () -> { for (int i = 0; i < 100; i++) { blackhole.consume(connection.getNamespaces().stream().count()); } diff --git a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/MemValueFactoryConcurrentBenchmark.java b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/MemValueFactoryConcurrentBenchmark.java index 4f5b02057cb..87c09670d95 100644 --- a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/MemValueFactoryConcurrentBenchmark.java +++ b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/MemValueFactoryConcurrentBenchmark.java @@ -38,6 +38,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -57,11 +58,14 @@ @State(Scope.Benchmark) @Warmup(iterations = 5) @BenchmarkMode({ Mode.AverageTime }) -@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) @Measurement(iterations = 5) @OutputTimeUnit(TimeUnit.MILLISECONDS) public class MemValueFactoryConcurrentBenchmark extends BaseConcurrentBenchmark { + @Param({ "10", "50", "100", "500", "1000", "5000" }) + public int numThreads; + public static final int BUCKET_SIZE = 10000; private SailRepository repository; private List> values; @@ -129,7 +133,7 @@ public void onlyReads(Blackhole blackhole) throws Exception { Random random = new Random(48593); - threads(100, () -> { + threads(numThreads, () -> { List values = this.values.get(random.nextInt(this.values.size())); @@ -151,7 +155,7 @@ public void readHeavy(Blackhole blackhole) throws Exception { Random random = new Random(48593); - threads(100, () -> { + threads(numThreads, () -> { Random r = new Random(random.nextInt()); for (int i = 0; i < BUCKET_SIZE; i++) { MemIRI orCreateMemURI = valueFactory @@ -171,7 +175,7 @@ public void onlyWrites(Blackhole blackhole) throws Exception { AtomicInteger atomicInteger = new AtomicInteger(); - threads(100, () -> { + threads(numThreads, () -> { int base = atomicInteger.incrementAndGet(); for (int i = 0; i < BUCKET_SIZE; i++) { IRI iri = valueFactory.createIRI("http://example.com", base + "-" + i); diff --git a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelMixedReadWriteBenchmark.java b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelMixedReadWriteBenchmark.java index 3e1bae0270c..8588011ff52 100644 --- a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelMixedReadWriteBenchmark.java +++ b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelMixedReadWriteBenchmark.java @@ -45,6 +45,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -67,12 +68,14 @@ @State(Scope.Benchmark) @Warmup(iterations = 5) @BenchmarkMode({ Mode.AverageTime }) -@Fork(value = 1, jvmArgs = { "-Xms64M", "-Xmx512M" }) -//@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", "-XX:StartFlightRecording=delay=20s,duration=120s,filename=recording.jfr,settings=profile", "-XX:FlightRecorderOptions=samplethreads=true,stackdepth=1024", "-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints" }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) @Measurement(iterations = 5) @OutputTimeUnit(TimeUnit.MILLISECONDS) public class ParallelMixedReadWriteBenchmark extends BaseConcurrentBenchmark { + @Param({ "1", "5", "10", "50", "100", "500" }) + public int workloadSize; + private static final String query1; private static final String query4; private static final String query7_pathexpression1; @@ -181,7 +184,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch RepositoryConnection connection, IsolationLevel isolationLevel) { ArrayList list = new ArrayList<>(); - for (int i = 0; i < 10; i++) { + for (int i = 0; i < workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query4) @@ -195,7 +198,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 3 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query7_pathexpression1) @@ -209,7 +212,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 3 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query8_pathexpression2) @@ -223,7 +226,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 400; i++) { + for (int i = 0; i < 40 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { blackhole.consume(localConnection.hasStatement(null, RDF.TYPE, null, false)); // System.out.println("Finished hasStatement explicit"); @@ -231,7 +234,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 400; i++) { + for (int i = 0; i < 40 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { blackhole.consume(localConnection.hasStatement(null, RDF.TYPE, null, true)); // System.out.println("Finished hasStatement inferred"); @@ -239,7 +242,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 2 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query1) @@ -252,7 +255,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 200; i++) { + for (int i = 0; i < 20 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { for (int j = 0; j < 100; j++) { localConnection.add(Values.bnode(), RDFS.LABEL, Values.literal(j), diff --git a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelQueryBenchmark.java b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelQueryBenchmark.java index d5dfcafd320..7f9c4c5b5c0 100644 --- a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelQueryBenchmark.java +++ b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ParallelQueryBenchmark.java @@ -39,6 +39,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -52,12 +53,14 @@ @State(Scope.Benchmark) @Warmup(iterations = 5) @BenchmarkMode({ Mode.AverageTime }) -@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G" }) -//@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", "-XX:StartFlightRecording=delay=20s,duration=120s,filename=recording.jfr,settings=profile", "-XX:FlightRecorderOptions=samplethreads=true,stackdepth=1024", "-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints" }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) @Measurement(iterations = 5) @OutputTimeUnit(TimeUnit.MILLISECONDS) public class ParallelQueryBenchmark extends BaseConcurrentBenchmark { + @Param({ "1", "5", "10", "50", "100", "500" }) + public int workloadSize; + private static final String query1; private static final String query4; private static final String query7_pathexpression1; @@ -133,7 +136,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch RepositoryConnection connection, IsolationLevel isolationLevel) { ArrayList list = new ArrayList<>(); - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 2 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query4) @@ -145,7 +148,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 2 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query7_pathexpression1) @@ -157,7 +160,7 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 2 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query8_pathexpression2) @@ -169,19 +172,19 @@ private ArrayList getMixedWorkload(Blackhole blackhole, CountDownLatch })); } - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 20 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { blackhole.consume(localConnection.hasStatement(null, RDF.TYPE, null, false)); })); } - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 20 * workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { blackhole.consume(localConnection.hasStatement(null, RDF.TYPE, null, true)); })); } - for (int i = 0; i < 5; i++) { + for (int i = 0; i < workloadSize; i++) { list.add(getRunnable(startSignal, connection, isolationLevel, (localConnection) -> { long count = localConnection .prepareTupleQuery(query1) diff --git a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/model/MemStatementListTestIT.java b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/model/MemStatementListTestIT.java index c8594d67184..85825e1da03 100644 --- a/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/model/MemStatementListTestIT.java +++ b/core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/model/MemStatementListTestIT.java @@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -47,17 +48,45 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import com.google.common.collect.Lists; @Tag("slow") +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@BenchmarkMode({ Mode.AverageTime }) +@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx50G", }) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public class MemStatementListTestIT { private static List statements; - public static final int CHUNKS = 1_000; + + @Param({ "10", "50", "100", "500", "1000", "5000" }) + public static int CHUNKS = 1_000; @BeforeAll + @Setup(Level.Trial) public static void beforeAll() throws IOException { + boolean isJmhRunning = System.getProperty("jmh.ignoreLock") != null; + if (!isJmhRunning) + CHUNKS = 1_000; MemoryStore memoryStore = new MemoryStore(); try { try (NotifyingSailConnection connection = memoryStore.getConnection()) { @@ -91,6 +120,7 @@ public static void beforeAll() throws IOException { } @BeforeEach + @Setup(Level.Iteration) public void beforeEach() { for (MemStatement statement : statements) { statement.setTillSnapshot(Integer.MAX_VALUE); @@ -100,6 +130,7 @@ public void beforeEach() { @Test @Timeout(120) + @Benchmark public void addMultipleThreads() throws ExecutionException, InterruptedException { List> partition = Lists.partition(statements, CHUNKS); @@ -140,6 +171,7 @@ public void addMultipleThreads() throws ExecutionException, InterruptedException @Test @Timeout(120) + @Benchmark public void addMultipleThreadsAndCleanupThread() throws ExecutionException, InterruptedException { List> partition = Lists.partition(statements, CHUNKS); diff --git a/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/benchmark/OverflowBenchmarkConcurrent.java b/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/benchmark/OverflowBenchmarkConcurrent.java index 277c6c498ec..bd3759639a8 100644 --- a/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/benchmark/OverflowBenchmarkConcurrent.java +++ b/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/benchmark/OverflowBenchmarkConcurrent.java @@ -53,6 +53,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -76,6 +77,9 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) public class OverflowBenchmarkConcurrent { + @Param({ "5", "25", "50", "250", "500", "2500" }) + public int numThreadsHalf; + @Setup(Level.Trial) public void setup() { ((Logger) (LoggerFactory @@ -115,7 +119,7 @@ public void manyConcurrentTransactions() throws IOException { CountDownLatch countDownLatch = new CountDownLatch(1); - for (int i = 0; i < 38; i++) { + for (int i = 0; i < numThreadsHalf; i++) { var seed = i + 485924; { Future submit = executorService.submit(() -> { diff --git a/run_jmh.sh b/run_jmh.sh new file mode 100755 index 00000000000..1ac748d95ea --- /dev/null +++ b/run_jmh.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +mvn clean package -Pbenchmarks -DskipTests +java -jar ./core/sail/memory/target/jmh-benchmarks.jar ConcurrentQueryBenchmark +java -jar ./core/sail/memory/target/jmh-benchmarks.jar MemStatementListTestIT +java -jar ./compliance/repository/target/jmh-benchmarks.jar RepositoryFederatedServiceIntegrationTest +java -jar ./core/queryalgebra/evaluation/target/jmh-benchmarks.jar MinimalContextNowTest +java -jar ./core/sail/lucene/target/jmh-benchmarks.jar LuceneSailTest +java -jar ./core/sail/memory/target/jmh-benchmarks.jar ParallelQueryBenchmark +java -jar ./core/sail/memory/target/jmh-benchmarks.jar ParallelMixedReadWriteBenchmark +java -jar ./core/sail/memory/target/jmh-benchmarks.jar MemValueFactoryConcurrentBenchmark + +java -jar ./core/sail/lmdb/target/jmh-benchmarks.jar OverflowBenchmarkConcurrent +java -jar ./core/sail/nativerdf/target/jmh-benchmarks.jar OverflowBenchmarkConcurrent