From f5553a286d5a7795771c79f91f9ae8a412cf4683 Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Thu, 20 Jul 2023 13:45:54 +0800 Subject: [PATCH 01/29] [fix][build] Upgrade PyYaml version to 6.0.1 (#20831) --- docker/pulsar/Dockerfile | 2 +- docker/pulsar/scripts/gen-yml-from-env.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index a5b294063d376..593401f57be8c 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -81,7 +81,7 @@ RUN apt-get -y --purge autoremove \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install pyyaml==5.4.1 +RUN pip3 install pyyaml==6.0.1 # Pulsar currently writes to the below directories, assuming the default configuration. # Note that number 4 is the reason that pulsar components need write access to the /pulsar directory. diff --git a/docker/pulsar/scripts/gen-yml-from-env.py b/docker/pulsar/scripts/gen-yml-from-env.py index ce19436b7e0dd..aa40408ed5b1f 100755 --- a/docker/pulsar/scripts/gen-yml-from-env.py +++ b/docker/pulsar/scripts/gen-yml-from-env.py @@ -61,7 +61,7 @@ conf_files = sys.argv[1:] for conf_filename in conf_files: - conf = yaml.load(open(conf_filename)) + conf = yaml.load(open(conf_filename), Loader=yaml.FullLoader) # update the config modified = False From 5d0aa5615df68ee6898d0c2e58006166b8a0a8bd Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 20 Jul 2023 17:48:33 +0800 Subject: [PATCH 02/29] [improve] [ws] add cryptoKeyReaderFactoryClassName into the file websocket.conf (#20840) Motivation: Since the PR https://github.com/apache/pulsar/pull/16234 add the prop `cryptoKeyReaderFactoryClassName` for the WebSocket Proxy, but did not add this prop to `websocket.conf`. This can make the script which try to replacement attribute a bit difficult to write Modifications: add the conf `cryptoKeyReaderFactoryClassName` into the file `websocket.conf` --- conf/websocket.conf | 3 +++ .../pulsar/websocket/WebSocketProxyConfigurationTest.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/conf/websocket.conf b/conf/websocket.conf index 2e2824a838c6f..490cff2722ee5 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -194,3 +194,6 @@ zooKeeperSessionTimeoutMillis=-1 # ZooKeeper cache expiry time in seconds # Deprecated: use metadataStoreCacheExpirySeconds zooKeeperCacheExpirySeconds=-1 + +# CryptoKeyReader factory classname to support encryption at websocket. +cryptoKeyReaderFactoryClassName= diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java index 92b4238cddd7d..a5ec63045bad3 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java @@ -34,6 +34,7 @@ import java.io.PrintWriter; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; public class WebSocketProxyConfigurationTest { @@ -64,6 +65,7 @@ public void testBackwardCompatibility() throws IOException { printWriter.println("metadataStoreCacheExpirySeconds=500"); printWriter.println("zooKeeperSessionTimeoutMillis=-1"); printWriter.println("zooKeeperCacheExpirySeconds=-1"); + printWriter.println("cryptoKeyReaderFactoryClassName="); } testConfigFile.deleteOnExit(); stream = new FileInputStream(testConfigFile); @@ -71,6 +73,7 @@ public void testBackwardCompatibility() throws IOException { stream.close(); assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 60); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 500); + assertNull(serviceConfig.getCryptoKeyReaderFactoryClassName()); testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); if (testConfigFile.exists()) { @@ -81,6 +84,7 @@ public void testBackwardCompatibility() throws IOException { printWriter.println("metadataStoreCacheExpirySeconds=30"); printWriter.println("zooKeeperSessionTimeoutMillis=100"); printWriter.println("zooKeeperCacheExpirySeconds=300"); + printWriter.println("cryptoKeyReaderFactoryClassName=A.class"); } testConfigFile.deleteOnExit(); stream = new FileInputStream(testConfigFile); @@ -88,6 +92,7 @@ public void testBackwardCompatibility() throws IOException { stream.close(); assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 100); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 300); + assertEquals(serviceConfig.getCryptoKeyReaderFactoryClassName(), "A.class"); } @Test From 514fa603788a9194e375915b8773e67268c5da78 Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Thu, 20 Jul 2023 19:25:20 +0900 Subject: [PATCH 03/29] [fix][sec] Suppress addressed CVE-2023-2976 in clickhouse-jdbc-0.4.6-all and canal.client-1.1.5 (#20792) --- src/owasp-dependency-check-suppressions.xml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 2cb82677db087..d5ddc28e884cb 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -109,7 +109,15 @@ b87878db57d5cfc2ca7d3972cc8f7486bf02fbca CVE-2020-8908 - + + + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2023-2976 + fa9a1ccda7d78edb51a3a33d3493566092786a30 CVE-2021-25263 + + + d3b929509399a698915b24ff47db781d0c526760 + CVE-2023-2976 + Date: Thu, 20 Jul 2023 21:07:43 +0800 Subject: [PATCH 04/29] [fix][broker] Fix direct memory leak by delayed index OutOfDirectMemory (#20823) --- .../common/util/collections/SegmentedLongArray.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java index 0b3520983b2ba..c551895c51a92 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java @@ -19,11 +19,11 @@ package org.apache.pulsar.common.util.collections; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import java.util.ArrayList; import java.util.List; import javax.annotation.concurrent.NotThreadSafe; import lombok.Getter; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; @NotThreadSafe public class SegmentedLongArray implements AutoCloseable { @@ -44,14 +44,14 @@ public SegmentedLongArray(long initialCapacity) { // Add first segment int sizeToAdd = (int) Math.min(remainingToAdd, MAX_SEGMENT_SIZE); - ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(sizeToAdd * SIZE_OF_LONG); + ByteBuf buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(sizeToAdd * SIZE_OF_LONG); buffer.writerIndex(sizeToAdd * SIZE_OF_LONG); buffers.add(buffer); remainingToAdd -= sizeToAdd; // Add the remaining segments, all at full segment size, if necessary while (remainingToAdd > 0) { - buffer = PooledByteBufAllocator.DEFAULT.directBuffer(MAX_SEGMENT_SIZE * SIZE_OF_LONG); + buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(MAX_SEGMENT_SIZE * SIZE_OF_LONG); buffer.writerIndex(MAX_SEGMENT_SIZE * SIZE_OF_LONG); buffers.add(buffer); remainingToAdd -= MAX_SEGMENT_SIZE; @@ -83,7 +83,7 @@ public void increaseCapacity() { } else { // Let's add 1 mode buffer to the list int bufferSize = MAX_SEGMENT_SIZE * SIZE_OF_LONG; - ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(bufferSize, bufferSize); + ByteBuf buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(bufferSize, bufferSize); buffer.writerIndex(bufferSize); buffers.add(buffer); capacity += MAX_SEGMENT_SIZE; @@ -107,7 +107,7 @@ public void shrink(long newCapacity) { // We should also reduce the capacity of the first buffer capacity -= sizeToReduce; ByteBuf oldBuffer = buffers.get(0); - ByteBuf newBuffer = PooledByteBufAllocator.DEFAULT.directBuffer((int) capacity * SIZE_OF_LONG); + ByteBuf newBuffer = PulsarByteBufAllocator.DEFAULT.directBuffer((int) capacity * SIZE_OF_LONG); oldBuffer.getBytes(0, newBuffer, (int) capacity * SIZE_OF_LONG); oldBuffer.release(); buffers.set(0, newBuffer); From ac94296d2488b2b13d76fa2b1bfa71e9e3cfbb45 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 20 Jul 2023 23:20:44 +0800 Subject: [PATCH 05/29] [improve][broker] Follow up #4196 use `PulsarByteBufAllocator` handle OOM (#20837) --- .../org/apache/pulsar/common/protocol/Markers.java | 12 ++++++------ .../impl/FileStoreBackedReadHandleImpl.java | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java index 50b036f99ee4b..2291aee781f60 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java @@ -19,13 +19,13 @@ package org.apache.pulsar.common.protocol; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.concurrent.FastThreadLocal; import java.io.IOException; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.MarkerType; import org.apache.pulsar.common.api.proto.MarkersMessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -109,7 +109,7 @@ public static ByteBuf newReplicatedSubscriptionsSnapshotRequest(String snapshotI .clear() .setSnapshotId(snapshotId) .setSourceCluster(sourceCluster); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(req.getSerializedSize()); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(req.getSerializedSize()); try { req.writeTo(payload); @@ -138,7 +138,7 @@ public static ByteBuf newReplicatedSubscriptionsSnapshotResponse(String snapshot .setLedgerId(ledgerId) .setEntryId(entryId); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(response.getSerializedSize()); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(response.getSerializedSize()); try { response.writeTo(payload); return newMessage(MarkerType.REPLICATED_SUBSCRIPTION_SNAPSHOT_RESPONSE, Optional.of(replyToCluster), @@ -172,7 +172,7 @@ public static ByteBuf newReplicatedSubscriptionsSnapshot(String snapshotId, Stri }); int size = snapshot.getSerializedSize(); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(size); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(size); try { snapshot.writeTo(payload); return newMessage(MarkerType.REPLICATED_SUBSCRIPTION_SNAPSHOT, Optional.of(sourceCluster), payload); @@ -201,7 +201,7 @@ public static ByteBuf newReplicatedSubscriptionsUpdate(String subscriptionName, .setMessageId().copyFrom(msgId); }); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(update.getSerializedSize()); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(update.getSerializedSize()); try { update.writeTo(payload); @@ -258,7 +258,7 @@ private static ByteBuf newTxnMarker(MarkerType markerType, long sequenceId, long .setTxnidMostBits(txnMostBits) .setTxnidLeastBits(txnLeastBits); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(0); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(0); try { return Commands.serializeMetadataAndPayload(ChecksumType.Crc32c, diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java index 506fbb8de68bf..49b2071f5db2c 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java @@ -20,7 +20,6 @@ import static org.apache.bookkeeper.mledger.offload.OffloadUtils.parseLedgerMetadata; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -40,6 +39,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.naming.TopicName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,7 +133,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr int length = value.getLength(); long entryId = key.get(); if (entryId == nextExpectedId) { - ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(length, length); + ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(length, length); entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf)); buf.writeBytes(value.copyBytes()); entriesToRead--; From d5f97bfae4dca33af3e2622306284e18ede63e7f Mon Sep 17 00:00:00 2001 From: crossoverJie Date: Fri, 21 Jul 2023 10:49:29 +0800 Subject: [PATCH 06/29] [improve][broker] Add broker interceptor conf (#20719) --- conf/broker.conf | 9 +++++++++ conf/standalone.conf | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/conf/broker.conf b/conf/broker.conf index 57b9f1bd38133..40eb1bcc32485 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1823,3 +1823,12 @@ managedLedgerCacheEvictionFrequency=0 # Number of worker threads to serve non-persistent topic # Deprecated - use topicOrderedExecutorThreadNum instead. numWorkerThreadsForNonPersistentTopic= + +# The directory to locate broker interceptors +brokerInterceptorsDirectory=./interceptors + +# List of broker interceptor to load, which is a list of broker interceptor names +brokerInterceptors= + +# Enable or disable the broker interceptor, which is only used for testing for now +disableBrokerInterceptors=true \ No newline at end of file diff --git a/conf/standalone.conf b/conf/standalone.conf index 46f4efbcf9171..76223c5933e45 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -1269,3 +1269,12 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 # after reaching the max buckets limitation, the adjacent buckets will be merged. # (disable with value -1) delayedDeliveryMaxNumBuckets=-1 + +# The directory to locate broker interceptors +brokerInterceptorsDirectory=./interceptors + +# List of broker interceptor to load, which is a list of broker interceptor names +brokerInterceptors= + +# Enable or disable the broker interceptor, which is only used for testing for now +disableBrokerInterceptors=true \ No newline at end of file From 9b6a1232cf35d15b9bf492f60f5e52534d879df2 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Fri, 21 Jul 2023 13:04:30 +0800 Subject: [PATCH 07/29] [fix][broker] Inconsistent behaviour for topic auto_creation (#20843) --- .../pulsar/broker/service/BrokerService.java | 16 ++-- .../impl/HierarchyTopicAutoCreationTest.java | 96 +++++++++++++++++++ 2 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 668195c4b8065..8dc94ca874094 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -3023,12 +3023,9 @@ public CompletableFuture fetchPartitionedTopicMetadata if (pulsar.getNamespaceService() == null) { return FutureUtil.failedFuture(new NamingException("namespace service is not ready")); } - Optional policies = - pulsar.getPulsarResources().getNamespaceResources() - .getPoliciesIfCached(topicName.getNamespaceObject()); - return pulsar.getNamespaceService().checkTopicExists(topicName) - .thenCompose(topicExists -> { - return fetchPartitionedTopicMetadataAsync(topicName) + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject()) + .thenCompose(policies -> pulsar.getNamespaceService().checkTopicExists(topicName) + .thenCompose(topicExists -> fetchPartitionedTopicMetadataAsync(topicName) .thenCompose(metadata -> { CompletableFuture future = new CompletableFuture<>(); @@ -3041,7 +3038,7 @@ public CompletableFuture fetchPartitionedTopicMetadata && !topicExists && !topicName.isPartitioned() && pulsar.getBrokerService() - .isDefaultTopicTypePartitioned(topicName, policies)) { + .isDefaultTopicTypePartitioned(topicName, policies)) { isAllowAutoTopicCreationAsync(topicName, policies).thenAccept(allowed -> { if (allowed) { pulsar.getBrokerService() @@ -3050,7 +3047,7 @@ public CompletableFuture fetchPartitionedTopicMetadata .exceptionally(ex -> { if (ex.getCause() instanceof MetadataStoreException - .AlreadyExistsException) { + .AlreadyExistsException) { // The partitioned topic might be created concurrently fetchPartitionedTopicMetadataAsync(topicName) .whenComplete((metadata2, ex2) -> { @@ -3078,8 +3075,7 @@ public CompletableFuture fetchPartitionedTopicMetadata }); return future; - }); - }); + }))); } @SuppressWarnings("deprecation") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java new file mode 100644 index 0000000000000..8c93b293c41a4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import lombok.Cleanup; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import java.util.List; +import java.util.UUID; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker-impl") +@Slf4j +public class HierarchyTopicAutoCreationTest extends ProducerConsumerBase { + + @Override + @BeforeMethod + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(invocationCount = 3) + @SneakyThrows + public void testPartitionedTopicAutoCreation() { + // Create namespace + final String namespace = "public/testPartitionedTopicAutoCreation"; + admin.namespaces().createNamespace(namespace); + // Set policies + final AutoTopicCreationOverride expectedPolicies = AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType("partitioned") + .defaultNumPartitions(1) + .build(); + admin.namespaces().setAutoTopicCreation(namespace, expectedPolicies); + // Double-check the policies + final AutoTopicCreationOverride nsAutoTopicCreationOverride = admin.namespaces() + .getAutoTopicCreation(namespace); + Assert.assertEquals(nsAutoTopicCreationOverride, expectedPolicies); + // Background invalidate cache + final MetadataCache nsCache = pulsar.getPulsarResources().getNamespaceResources().getCache(); + final Thread t1 = new Thread(() -> { + while (true) { + nsCache.invalidate("/admin/policies/" + namespace); + } + }); + t1.start(); + + // trigger auto-creation + final String topicName = "persistent://" + namespace + "/test-" + UUID.randomUUID(); + @Cleanup final Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + final List topics = admin.topics().getList(namespace); + Assert.assertEquals(topics.size(), 1); // expect only one topic + Assert.assertEquals(topics.get(0), + TopicName.get(topicName).getPartition(0).toString()); // expect partitioned topic + + // double-check policies + final AutoTopicCreationOverride actualPolicies2 = admin.namespaces().getAutoTopicCreation(namespace); + Assert.assertEquals(actualPolicies2, expectedPolicies); + + t1.interrupt(); + } +} From b8e6948f62d6ec2ca53b6a85fe2fd07d4dee6853 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Fri, 21 Jul 2023 15:41:50 -0700 Subject: [PATCH 08/29] [fix][broker] Broker failed to load v1 namespace resources cache (#20783) --- .../broker/resources/BaseResources.java | 36 +++++++++++++++++++ .../broker/resources/NamespaceResources.java | 2 +- .../broker/admin/PersistentTopicsTest.java | 14 ++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java index 42add4271f684..4011a48207512 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java @@ -20,11 +20,16 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -78,6 +83,37 @@ protected CompletableFuture> getChildrenAsync(String path) { return cache.getChildren(path); } + protected CompletableFuture> getChildrenRecursiveAsync(String path) { + Set children = ConcurrentHashMap.newKeySet(); + CompletableFuture> result = new CompletableFuture<>(); + getChildrenRecursiveAsync(path, children, result, new AtomicInteger(1), path); + return result; + } + + private void getChildrenRecursiveAsync(String path, Set children, CompletableFuture> result, + AtomicInteger totalResults, String parent) { + cache.getChildren(path).thenAccept(childList -> { + childList = childList != null ? childList : Collections.emptyList(); + if (totalResults.decrementAndGet() == 0 && childList.isEmpty()) { + result.complete(new ArrayList<>(children)); + return; + } + if (childList.isEmpty()) { + return; + } + // remove current node from children if current node is not leaf + children.remove(parent); + // childPrefix creates a path hierarchy if children has multi level path + String childPrefix = path.equals(parent) ? "" : parent + "/"; + totalResults.addAndGet(childList.size()); + for (String child : childList) { + children.add(childPrefix + child); + String childPath = path + "/" + child; + getChildrenRecursiveAsync(childPath, children, result, totalResults, child); + } + }); + } + protected Optional get(String path) throws MetadataStoreException { try { return getAsync(path).get(operationTimeoutSec, TimeUnit.SECONDS); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index 48f8259656729..e5dd13c32eb23 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -62,7 +62,7 @@ public NamespaceResources(MetadataStore configurationStore, int operationTimeout } public CompletableFuture> listNamespacesAsync(String tenant) { - return getChildrenAsync(joinPath(BASE_POLICIES_PATH, tenant)); + return getChildrenRecursiveAsync(joinPath(BASE_POLICIES_PATH, tenant)); } public CompletableFuture getPoliciesReadOnlyAsync() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 83ed63bf0d95d..a4f6bd4650f7c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; @@ -1700,4 +1701,17 @@ public void testInternalGetReplicatedSubscriptionStatusFromLocal() throws Except // verify we only call getReplicatedSubscriptionStatusAsync once. verify(topics, times(1)).getReplicatedSubscriptionStatusAsync(any(), any()); } + + @Test + public void testNamespaceResources() throws Exception { + String ns1V1 = "test/" + testNamespace + "v1"; + String ns1V2 = testNamespace + "v2"; + admin.namespaces().createNamespace(testTenant+"/"+ns1V1); + admin.namespaces().createNamespace(testTenant+"/"+ns1V2); + + List namespaces = pulsar.getPulsarResources().getNamespaceResources().listNamespacesAsync(testTenant) + .get(); + assertTrue(namespaces.contains(ns1V2)); + assertTrue(namespaces.contains(ns1V1)); + } } From 9ccc562e138d4caadad27014cc683f689328db34 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 22 Jul 2023 17:55:30 +0800 Subject: [PATCH 09/29] [fix][broker] Pass `bytesToRead` when reading compacted entries (#20850) --- ...PersistentDispatcherSingleActiveConsumer.java | 4 ++-- .../apache/pulsar/compaction/CompactedTopic.java | 4 ++-- .../pulsar/compaction/CompactedTopicUtils.java | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 6de113d6db941..d96429693fda8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -350,8 +350,8 @@ protected void readMoreEntries(Consumer consumer) { havePendingRead = true; if (consumer.readCompacted()) { boolean readFromEarliest = isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId()); - CompactedTopicUtils.readCompactedEntries(topic.getTopicCompactionService(), cursor, messagesToRead, - readFromEarliest, this, consumer); + CompactedTopicUtils.asyncReadCompactedEntries(topic.getTopicCompactionService(), cursor, + messagesToRead, bytesToRead, readFromEarliest, this, true, consumer); } else { ReadEntriesCtx readEntriesCtx = ReadEntriesCtx.create(consumer, consumer.getConsumerEpoch()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java index 99e2f8a9624e8..660c7ea779741 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java @@ -33,8 +33,8 @@ public interface CompactedTopic { /** * Read entries from compacted topic. * - * @deprecated Use {@link CompactedTopicUtils#readCompactedEntries(TopicCompactionService, ManagedCursor, - * int, boolean, ReadEntriesCallback, Consumer)} instead. + * @deprecated Use {@link CompactedTopicUtils#asyncReadCompactedEntries(TopicCompactionService, ManagedCursor, + * int, long, boolean, ReadEntriesCallback, boolean, Consumer)} instead. */ @Deprecated void asyncReadEntriesOrWait(ManagedCursor cursor, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java index 4cd21cbb03eea..cc5147c8e6623 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java @@ -39,9 +39,11 @@ public class CompactedTopicUtils { @Beta - public static void readCompactedEntries(TopicCompactionService topicCompactionService, ManagedCursor cursor, - int numberOfEntriesToRead, boolean readFromEarliest, - AsyncCallbacks.ReadEntriesCallback callback, @Nullable Consumer consumer) { + public static void asyncReadCompactedEntries(TopicCompactionService topicCompactionService, + ManagedCursor cursor, int numberOfEntriesToRead, + long bytesToRead, boolean readFromEarliest, + AsyncCallbacks.ReadEntriesCallback callback, + boolean wait, @Nullable Consumer consumer) { Objects.requireNonNull(topicCompactionService); Objects.requireNonNull(cursor); checkArgument(numberOfEntriesToRead > 0); @@ -64,7 +66,13 @@ public static void readCompactedEntries(TopicCompactionService topicCompactionSe if (lastCompactedPosition == null || readPosition.compareTo( lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) > 0) { - cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, PositionImpl.LATEST); + if (wait) { + cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, bytesToRead, callback, readEntriesCtx, + PositionImpl.LATEST); + } else { + cursor.asyncReadEntries(numberOfEntriesToRead, bytesToRead, callback, readEntriesCtx, + PositionImpl.LATEST); + } return CompletableFuture.completedFuture(null); } From ebfef6de5e2026aa2226a4776dba07df0897df28 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 22 Jul 2023 19:06:04 +0800 Subject: [PATCH 10/29] [optimize][admin]Enhancing Transaction Buffer Stats and Introducing TransactionBufferInternalStats API (#20330) master https://github.com/apache/pulsar/issues/20291 ### Motivation Our primary goal is to improve the visibility and troubleshooting capabilities of the Pulsar Transaction Buffer by providing more detailed information about the snapshot stats and system topic internal status. ### Modifications 1. Enhance the existing TransactionBufferStats by adding information about snapshot stats, including the capital of the current segment, unseal aborted transaction ID size, and other related data. This will provide better visibility and troubleshooting capabilities for the Pulsar Transaction Buffer. 2. Introduce a new API for obtaining TransactionBufferInternalStats, allowing users to access the state of the system topic used for storing snapshots. This will facilitate problem investigation and resolution when issues arise with the transaction buffer. --- .../broker/admin/impl/TransactionsBase.java | 69 ++++++++++- .../pulsar/broker/admin/v3/Transactions.java | 68 +++++++++-- .../service/persistent/PersistentTopic.java | 6 +- .../buffer/AbortedTxnProcessor.java | 11 +- .../transaction/buffer/TransactionBuffer.java | 17 +++ .../buffer/impl/InMemTransactionBuffer.java | 14 ++- ...SingleSnapshotAbortedTxnProcessorImpl.java | 8 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 28 ++++- .../buffer/impl/TopicTransactionBuffer.java | 21 +++- .../buffer/impl/TransactionBufferDisable.java | 11 ++ .../AdminApiTransactionMultiBrokerTest.java | 46 ++++++++ .../admin/v3/AdminApiTransactionTest.java | 107 +++++++++++++++++- .../SegmentAbortedTxnProcessorTest.java | 88 +++++++++++++- .../broker/transaction/TransactionTest.java | 4 +- .../transaction/TransactionTestBase.java | 3 +- .../buffer/TransactionStablePositionTest.java | 4 +- .../pulsar/client/admin/Transactions.java | 63 ++++++++++- .../common/policies/data/SegmentStats.java | 32 ++++++ .../common/policies/data/SegmentsStats.java | 37 ++++++ .../SnapshotSystemTopicInternalStats.java | 27 +++++ .../data/TransactionBufferInternalStats.java | 33 ++++++ .../policies/data/TransactionBufferStats.java | 9 ++ .../admin/internal/TransactionsImpl.java | 26 ++++- .../pulsar/admin/cli/PulsarAdminToolTest.java | 6 +- .../pulsar/admin/cli/CmdTransactions.java | 22 +++- 25 files changed, 713 insertions(+), 47 deletions(-) create mode 100644 pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java create mode 100644 pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java create mode 100644 pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java create mode 100644 pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index d596cbdd39db9..3921334cff30a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; @@ -38,6 +39,7 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.Transactions; @@ -47,6 +49,8 @@ import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.SnapshotSystemTopicInternalStats; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -170,9 +174,10 @@ protected CompletableFuture internalGetTransactionInBu } protected CompletableFuture internalGetTransactionBufferStats(boolean authoritative, - boolean lowWaterMarks) { + boolean lowWaterMarks, + boolean segmentStats) { return getExistingPersistentTopicAsync(authoritative) - .thenApply(topic -> topic.getTransactionBufferStats(lowWaterMarks)); + .thenApply(topic -> topic.getTransactionBufferStats(lowWaterMarks, segmentStats)); } protected CompletableFuture internalGetPendingAckStats( @@ -431,6 +436,66 @@ protected CompletableFuture internalGetPendi ); } + protected CompletableFuture internalGetTransactionBufferInternalStats( + boolean authoritative, boolean metadata) { + TransactionBufferInternalStats transactionBufferInternalStats = new TransactionBufferInternalStats(); + return getExistingPersistentTopicAsync(authoritative) + .thenCompose(topic -> { + AbortedTxnProcessor.SnapshotType snapshotType = topic.getTransactionBuffer().getSnapshotType(); + if (snapshotType == null) { + return FutureUtil.failedFuture(new RestException(NOT_FOUND, + "Transaction buffer Snapshot for the topic does not exist")); + } else if (snapshotType == AbortedTxnProcessor.SnapshotType.Segment) { + transactionBufferInternalStats.snapshotType = snapshotType.toString(); + TopicName segmentTopic = TopicName.get(TopicDomain.persistent.toString(), namespaceName, + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + CompletableFuture segmentInternalStatsFuture = + getTxnSnapshotInternalStats(segmentTopic, metadata); + TopicName indexTopic = TopicName.get(TopicDomain.persistent.toString(), + namespaceName, + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + CompletableFuture segmentIndexInternalStatsFuture = + getTxnSnapshotInternalStats(indexTopic, metadata); + return segmentIndexInternalStatsFuture + .thenCombine(segmentInternalStatsFuture, (indexStats, segmentStats) -> { + transactionBufferInternalStats.segmentIndexInternalStats = indexStats; + transactionBufferInternalStats.segmentInternalStats = segmentStats; + return transactionBufferInternalStats; + }); + } else if (snapshotType == AbortedTxnProcessor.SnapshotType.Single) { + transactionBufferInternalStats.snapshotType = snapshotType.toString(); + TopicName singleSnapshotTopic = TopicName.get(TopicDomain.persistent.toString(), namespaceName, + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + return getTxnSnapshotInternalStats(singleSnapshotTopic, metadata) + .thenApply(snapshotSystemTopicInternalStats -> { + transactionBufferInternalStats.singleSnapshotSystemTopicInternalStats = + snapshotSystemTopicInternalStats; + return transactionBufferInternalStats; + }); + } + return FutureUtil.failedFuture(new RestException(INTERNAL_SERVER_ERROR, "Unknown SnapshotType " + + snapshotType)); + }); + } + + private CompletableFuture getTxnSnapshotInternalStats(TopicName topicName, + boolean metadata) { + final PulsarAdmin admin; + try { + admin = pulsar().getAdminClient(); + } catch (PulsarServerException e) { + return FutureUtil.failedFuture(new RestException(e)); + } + return admin.topics().getInternalStatsAsync(topicName.toString(), metadata) + .thenApply(persistentTopicInternalStats -> { + SnapshotSystemTopicInternalStats + snapshotSystemTopicInternalStats = new SnapshotSystemTopicInternalStats(); + snapshotSystemTopicInternalStats.managedLedgerInternalStats = persistentTopicInternalStats; + snapshotSystemTopicInternalStats.managedLedgerName = topicName.getEncodedLocalName(); + return snapshotSystemTopicInternalStats; + }); + } + protected CompletableFuture getExistingPersistentTopicAsync(boolean authoritative) { return validateTopicOwnershipAsync(topicName, authoritative).thenCompose(__ -> { CompletableFuture> topicFuture = pulsar().getBrokerService() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java index aa24dbdcc3ae9..1bdc2255085f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java @@ -44,6 +44,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.common.util.FutureUtil; +import org.jetbrains.annotations.Nullable; @Path("/transactions") @Produces(MediaType.APPLICATION_JSON) @@ -171,11 +172,13 @@ public void getTransactionBufferStats(@Suspended final AsyncResponse asyncRespon @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("lowWaterMarks") @DefaultValue("false") - boolean lowWaterMarks) { + boolean lowWaterMarks, + @QueryParam("segmentStats") @DefaultValue("false") + boolean segmentStats) { try { checkTransactionCoordinatorEnabled(); validateTopicName(tenant, namespace, encodedTopic); - internalGetTransactionBufferStats(authoritative, lowWaterMarks) + internalGetTransactionBufferStats(authoritative, lowWaterMarks, segmentStats) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { if (!isRedirectException(ex)) { @@ -315,17 +318,58 @@ public void getPendingAckInternalStats(@Suspended final AsyncResponse asyncRespo log.error("[{}] Failed to get pending ack internal stats {}", clientAppId(), topicName, ex); } - Throwable cause = FutureUtil.unwrapCompletionException(ex); - if (cause instanceof BrokerServiceException.ServiceUnitNotReadyException) { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, cause)); - } else if (cause instanceof BrokerServiceException.NotAllowedException) { - asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, cause)); - } else if (cause instanceof BrokerServiceException.SubscriptionNotFoundException) { - asyncResponse.resume(new RestException(NOT_FOUND, cause)); - } else { - asyncResponse.resume(new RestException(cause)); + return resumeAsyncResponseWithBrokerException(asyncResponse, ex); + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } + } + + @Nullable + private Void resumeAsyncResponseWithBrokerException(@Suspended AsyncResponse asyncResponse, + Throwable ex) { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof BrokerServiceException.ServiceUnitNotReadyException) { + asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, cause)); + } else if (cause instanceof BrokerServiceException.NotAllowedException) { + asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, cause)); + } else if (cause instanceof BrokerServiceException.SubscriptionNotFoundException) { + asyncResponse.resume(new RestException(NOT_FOUND, cause)); + } else { + asyncResponse.resume(new RestException(cause)); + } + return null; + } + + @GET + @Path("/transactionBufferInternalStats/{tenant}/{namespace}/{topic}") + @ApiOperation(value = "Get transaction buffer internal stats.") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), + @ApiResponse(code = 503, message = "This Broker is not enable transaction"), + @ApiResponse(code = 307, message = "Topic is not owned by this broker!"), + @ApiResponse(code = 405, message = "Transaction buffer don't use managedLedger!"), + @ApiResponse(code = 400, message = "Topic is not a persistent topic!"), + @ApiResponse(code = 409, message = "Concurrent modification") + }) + public void getTransactionBufferInternalStats(@Suspended final AsyncResponse asyncResponse, + @QueryParam("authoritative") + @DefaultValue("false") boolean authoritative, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("metadata") @DefaultValue("false") boolean metadata) { + try { + validateTopicName(tenant, namespace, encodedTopic); + internalGetTransactionBufferInternalStats(authoritative, metadata) + .thenAccept(asyncResponse::resume) + .exceptionally(ex -> { + if (!isRedirectException(ex)) { + log.error("[{}] Failed to get transaction buffer internal stats {}", + clientAppId(), topicName, ex); } - return null; + return resumeAsyncResponseWithBrokerException(asyncResponse, ex); }); } catch (Exception ex) { resumeAsyncResponseExceptionally(asyncResponse, ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1e055eccc42f6..e907caa8c3000 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3566,7 +3566,11 @@ public boolean checkSubscriptionTypesEnable(SubType subType) { } public TransactionBufferStats getTransactionBufferStats(boolean lowWaterMarks) { - return this.transactionBuffer.getStats(lowWaterMarks); + return getTransactionBufferStats(lowWaterMarks, false); + } + + public TransactionBufferStats getTransactionBufferStats(boolean lowWaterMarks, boolean segmentStats) { + return this.transactionBuffer.getStats(lowWaterMarks, segmentStats); } public TransactionPendingAckStats getTransactionPendingAckStats(String subName, boolean lowWaterMarks) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 8223aa12b75ae..0f06c201a81ee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -21,10 +21,16 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; public interface AbortedTxnProcessor { + enum SnapshotType { + Single, + Segment, + } + /** * After the transaction buffer writes a transaction aborted marker to the topic, * the transaction buffer will put the aborted txnID and the aborted marker position to AbortedTxnProcessor. @@ -66,9 +72,10 @@ public interface AbortedTxnProcessor { /** * Get the lastSnapshotTimestamps. - * @return the lastSnapshotTimestamps. + * + * @return a transactionBufferStats with the stats in the abortedTxnProcessor. */ - long getLastSnapshotTimestamps(); + TransactionBufferStats generateSnapshotStats(boolean segmentStats); CompletableFuture closeAsync(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java index 9c32f7629969f..7eb5d6f789c22 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java @@ -158,12 +158,29 @@ public interface TransactionBuffer { */ PositionImpl getMaxReadPosition(); + /** + * Get the snapshot type. + * + * The snapshot type can be either "Single" or "Segment". In "Single" mode, a single snapshot log is used + * to record the transaction buffer stats. In "Segment" mode, a snapshot segment topic is used to record + * the stats, and a separate snapshot segment index topic is used to index these stats. + * + * @return the snapshot type + */ + AbortedTxnProcessor.SnapshotType getSnapshotType(); + /** * Get transaction in buffer stats. * @return the transaction in buffer stats. */ TransactionInBufferStats getTransactionInBufferStats(TxnID txnID); + /** + * Get transaction stats in buffer. + * @return the transaction stats in buffer. + */ + TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats); + /** * Get transaction stats in buffer. * @return the transaction stats in buffer. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java index 56b49f98efe8e..bc2dd58a5812e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java @@ -33,6 +33,7 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; @@ -374,16 +375,27 @@ public PositionImpl getMaxReadPosition() { return PositionImpl.LATEST; } + @Override + public AbortedTxnProcessor.SnapshotType getSnapshotType() { + return null; + } + @Override public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { return null; } @Override - public TransactionBufferStats getStats(boolean lowWaterMarks) { + public TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats) { return null; } + @Override + public TransactionBufferStats getStats(boolean lowWaterMarks) { + return getStats(lowWaterMarks, false); + } + + @Override public CompletableFuture checkIfTBRecoverCompletely(boolean isTxn) { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 5d582d564eadd..967f1f16fefe4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -173,8 +174,11 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit } @Override - public long getLastSnapshotTimestamps() { - return this.lastSnapshotTimestamps; + public TransactionBufferStats generateSnapshotStats(boolean segmentStats) { + TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); + transactionBufferStats.lastSnapshotTimestamps = this.lastSnapshotTimestamps; + transactionBufferStats.totalAbortedTransactions = aborts.size(); + return transactionBufferStats; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index cd6c9c6123a5b..be1271a155cd3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -62,6 +62,9 @@ import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SegmentStats; +import org.apache.pulsar.common.policies.data.SegmentsStats; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.FutureUtil; @@ -116,6 +119,8 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private volatile long lastSnapshotTimestamps; + private volatile long lastTakedSnapshotSegmentTimestamp; + /** * The number of the aborted transaction IDs in a segment. * This is calculated according to the configured memory size. @@ -451,9 +456,25 @@ public CompletableFuture clearAbortedTxnSnapshot() { persistentWorker::clearSnapshotSegmentAndIndexes); } - @Override - public long getLastSnapshotTimestamps() { - return this.lastSnapshotTimestamps; + public TransactionBufferStats generateSnapshotStats(boolean segmentStats) { + TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); + transactionBufferStats.totalAbortedTransactions = this.aborts.size(); + transactionBufferStats.lastSnapshotTimestamps = this.lastSnapshotTimestamps; + SegmentsStats segmentsStats = new SegmentsStats(); + segmentsStats.currentSegmentCapacity = this.snapshotSegmentCapacity; + segmentsStats.lastTookSnapshotSegmentTimestamp = this.lastTakedSnapshotSegmentTimestamp; + segmentsStats.unsealedAbortTxnIDSize = this.unsealedTxnIds.size(); + segmentsStats.segmentsSize = indexes.size(); + if (segmentStats) { + List statsList = new ArrayList<>(); + segmentIndex.forEach((position, txnID) -> { + SegmentStats stats = new SegmentStats(txnID.toString(), position.toString()); + statsList.add(stats); + }); + segmentsStats.segmentStats = statsList; + } + transactionBufferStats.segmentsStats = segmentsStats; + return transactionBufferStats; } @Override @@ -705,6 +726,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); }).thenCompose((messageId) -> { + lastTakedSnapshotSegmentTimestamp = System.currentTimeMillis(); //Build index for this segment TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 89a8e95afba1f..3c13be220869f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -101,6 +101,8 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final AbortedTxnProcessor snapshotAbortedTxnProcessor; + private final AbortedTxnProcessor.SnapshotType snapshotType; + public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; @@ -112,8 +114,10 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); + snapshotType = AbortedTxnProcessor.SnapshotType.Segment; } else { snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + snapshotType = AbortedTxnProcessor.SnapshotType.Single; } this.recover(); } @@ -489,6 +493,11 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { } } + @Override + public AbortedTxnProcessor.SnapshotType getSnapshotType() { + return snapshotType; + } + @Override public PositionImpl getMaxReadPosition() { if (checkIfReady() || checkIfNoSnapshot()) { @@ -509,9 +518,10 @@ public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { } @Override - public TransactionBufferStats getStats(boolean lowWaterMarks) { - TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); - transactionBufferStats.lastSnapshotTimestamps = this.snapshotAbortedTxnProcessor.getLastSnapshotTimestamps(); + public TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats) { + TransactionBufferStats transactionBufferStats = this.snapshotAbortedTxnProcessor + .generateSnapshotStats(segmentStats); + transactionBufferStats.snapshotType = snapshotType.toString(); transactionBufferStats.state = this.getState().name(); transactionBufferStats.maxReadPosition = this.maxReadPosition.toString(); if (lowWaterMarks) { @@ -524,6 +534,11 @@ public TransactionBufferStats getStats(boolean lowWaterMarks) { return transactionBufferStats; } + @Override + public TransactionBufferStats getStats(boolean lowWaterMarks) { + return getStats(lowWaterMarks, false); + } + @Override public void run(Timeout timeout) { if (checkIfReady()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java index 22ba8e2d2e8ef..7c74b52951e28 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java @@ -25,6 +25,7 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; @@ -93,11 +94,21 @@ public PositionImpl getMaxReadPosition() { return PositionImpl.LATEST; } + @Override + public AbortedTxnProcessor.SnapshotType getSnapshotType() { + return null; + } + @Override public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { return null; } + @Override + public TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats) { + return null; + } + @Override public TransactionBufferStats getStats(boolean lowWaterMarks) { return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java index 52aadde7b2621..bf51c69fbaef3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java @@ -19,13 +19,22 @@ package org.apache.pulsar.broker.admin.v3; import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import java.util.Map; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -81,4 +90,41 @@ public void testRedirectOfGetCoordinatorInternalStats() throws Exception { localAdmin.transactions().getCoordinatorInternalStats(i, false); } } + + @Test + public void testGetTransactionBufferInternalStatsInMultiBroker() throws Exception { + for (int i = 0; i < super.getBrokerCount(); i++) { + getPulsarServiceList().get(i).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + } + String topic1 = NAMESPACE1 + "/testGetTransactionBufferInternalStatsInMultiBroker"; + assertTrue(admin.namespaces().getBundles(NAMESPACE1).getNumBundles() > 1); + for (int i = 0; true ; i++) { + topic1 = topic1 + i; + admin.topics().createNonPartitionedTopic(topic1); + String segmentTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + String indexTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + if (segmentTopicBroker.equals(indexTopicBroker)) { + String topicBroker = admin.lookups().lookupTopic(topic1); + if (!topicBroker.equals(segmentTopicBroker)) { + break; + } + } else { + break; + } + } + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic1).create(); + TransactionBufferInternalStats stats = admin.transactions() + .getTransactionBufferInternalStatsAsync(topic1, true).get(); + assertEquals(stats.snapshotType, AbortedTxnProcessor.SnapshotType.Segment.toString()); + assertNull(stats.singleSnapshotSystemTopicInternalStats); + assertNotNull(stats.segmentInternalStats); + assertTrue(stats.segmentInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS)); + assertNotNull(stats.segmentIndexInternalStats); + assertTrue(stats.segmentIndexInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 0e51470da75a5..1e5f4679492ad 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -40,6 +40,7 @@ import org.apache.http.HttpStatus; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -60,7 +61,9 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -510,7 +513,7 @@ public void testGetCoordinatorInternalStats() throws Exception { TransactionCoordinatorInternalStats stats = admin.transactions() .getCoordinatorInternalStatsAsync(0, true).get(); - verifyManagedLegerInternalStats(stats.transactionLogStats.managedLedgerInternalStats, 26); + verifyManagedLedgerInternalStats(stats.transactionLogStats.managedLedgerInternalStats, 26); assertEquals(TopicName.get(TopicDomain.persistent.toString(), NamespaceName.SYSTEM_NAMESPACE, MLTransactionLogImpl.TRANSACTION_LOG_PREFIX + "0").getPersistenceNamingEncoding(), stats.transactionLogStats.managedLedgerName); @@ -565,7 +568,7 @@ public void testGetPendingAckInternalStats() throws Exception { + subName + SystemTopicNames.PENDING_ACK_STORE_SUFFIX).getPersistenceNamingEncoding(), stats.pendingAckLogStats.managedLedgerName); - verifyManagedLegerInternalStats(managedLedgerInternalStats, 16); + verifyManagedLedgerInternalStats(managedLedgerInternalStats, 16); ManagedLedgerInternalStats finalManagedLedgerInternalStats = managedLedgerInternalStats; managedLedgerInternalStats.cursors.forEach((s, cursorStats) -> { @@ -584,6 +587,88 @@ public void testGetPendingAckInternalStats() throws Exception { assertNull(managedLedgerInternalStats.ledgers.get(0).metadata); } + @Test(timeOut = 20000) + public void testGetTransactionBufferInternalStats() throws Exception { + // Initialize transaction + initTransaction(1); + + // Create topics + final String topic1 = "persistent://public/default/testGetTransactionBufferInternalStats-1"; + final String topic2 = "persistent://public/default/testGetTransactionBufferInternalStats-2"; + final String topic3 = "persistent://public/default/testGetTransactionBufferInternalStats-3"; + pulsar.getConfig().setTransactionCoordinatorEnabled(false); + admin.topics().createNonPartitionedTopic(topic1); + + // Verify NotFoundException when transaction coordinator is disabled + try { + admin.transactions().getTransactionBufferInternalStatsAsync(topic1, true).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarAdminException.NotFoundException); + } + + // Enable transaction coordinator and disable segmented snapshot + pulsar.getConfig().setTransactionCoordinatorEnabled(true); + pulsar.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); + + // Send a message with a transaction and abort it + Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic2).create(); + TransactionImpl transaction = (TransactionImpl) getTransaction(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + + // Get transaction buffer internal stats and verify single snapshot stats + TransactionBufferInternalStats stats = admin.transactions() + .getTransactionBufferInternalStatsAsync(topic2, true).get(); + assertEquals(stats.snapshotType, AbortedTxnProcessor.SnapshotType.Single.toString()); + assertNotNull(stats.singleSnapshotSystemTopicInternalStats); + + // Get managed ledger internal stats for the transaction buffer snapshot topic + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats( + TopicName.get(topic2).getNamespace() + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + verifyManagedLedgerInternalStats(stats.singleSnapshotSystemTopicInternalStats.managedLedgerInternalStats, + internalStats); + assertTrue(stats.singleSnapshotSystemTopicInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT)); + assertNull(stats.segmentInternalStats); + assertNull(stats.segmentIndexInternalStats); + + // Configure segmented snapshot and set segment size + pulsar.getConfig().setTransactionBufferSnapshotSegmentSize(9); + pulsar.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + + // Send a message with a transaction and abort it + producer = pulsarClient.newProducer(Schema.BYTES).topic(topic3).create(); + transaction = (TransactionImpl) getTransaction(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + + // Get transaction buffer internal stats and verify segmented snapshot stats + stats = admin.transactions().getTransactionBufferInternalStatsAsync(topic3, true).get(); + assertEquals(stats.snapshotType, AbortedTxnProcessor.SnapshotType.Segment.toString()); + assertNull(stats.singleSnapshotSystemTopicInternalStats); + assertNotNull(stats.segmentInternalStats); + + // Get managed ledger internal stats for the transaction buffer segments topic + internalStats = admin.topics().getInternalStats( + TopicName.get(topic2).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + verifyManagedLedgerInternalStats(stats.segmentInternalStats.managedLedgerInternalStats, internalStats); + assertTrue(stats.segmentInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS)); + + // Get managed ledger internal stats for the transaction buffer indexes topic + assertNotNull(stats.segmentIndexInternalStats); + internalStats = admin.topics().getInternalStats( + TopicName.get(topic2).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + verifyManagedLedgerInternalStats(stats.segmentIndexInternalStats.managedLedgerInternalStats, internalStats); + assertTrue(stats.segmentIndexInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES)); + } + + + @Test(timeOut = 20000) public void testTransactionNotEnabled() throws Exception { cleanup(); @@ -836,7 +921,7 @@ private Transaction getTransaction() throws Exception { .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); } - private static void verifyManagedLegerInternalStats(ManagedLedgerInternalStats managedLedgerInternalStats, + private static void verifyManagedLedgerInternalStats(ManagedLedgerInternalStats managedLedgerInternalStats, long totalSize) { assertEquals(managedLedgerInternalStats.entriesAddedCounter, 1); assertEquals(managedLedgerInternalStats.numberOfEntries, 1); @@ -851,4 +936,20 @@ private static void verifyManagedLegerInternalStats(ManagedLedgerInternalStats m assertNotNull(managedLedgerInternalStats.ledgers.get(0).metadata); assertEquals(managedLedgerInternalStats.cursors.size(), 1); } + + private static void verifyManagedLedgerInternalStats(ManagedLedgerInternalStats internalStats, + ManagedLedgerInternalStats persistentTopicStats) { + assertEquals(persistentTopicStats.entriesAddedCounter, internalStats.entriesAddedCounter); + assertEquals(persistentTopicStats.numberOfEntries, internalStats.numberOfEntries); + assertEquals(persistentTopicStats.totalSize, internalStats.totalSize); + assertEquals(persistentTopicStats.currentLedgerEntries, internalStats.currentLedgerEntries); + assertEquals(persistentTopicStats.currentLedgerSize, internalStats.currentLedgerSize); + assertEquals(persistentTopicStats.lastLedgerCreationFailureTimestamp, internalStats.lastLedgerCreationFailureTimestamp); + assertEquals(persistentTopicStats.waitingCursorsCount, internalStats.waitingCursorsCount); + assertEquals(persistentTopicStats.pendingAddEntriesCount, internalStats.pendingAddEntriesCount); + assertEquals(persistentTopicStats.lastConfirmedEntry, internalStats.lastConfirmedEntry); + assertNotNull(internalStats.ledgers.get(0).metadata); + assertEquals(persistentTopicStats.ledgers.size(), internalStats.ledgers.size()); + assertEquals(persistentTopicStats.cursors.size(), internalStats.cursors.size()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index 9e1c1992477af..6e763cb44fbc8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -18,9 +18,11 @@ */ package org.apache.pulsar.broker.transaction; -import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; import java.lang.reflect.Field; import java.util.LinkedList; import java.util.NavigableMap; @@ -29,6 +31,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -47,8 +50,9 @@ import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; -import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.Transactions; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.transaction.Transaction; @@ -56,6 +60,7 @@ import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.awaitility.reflect.WhiteboxImpl; @@ -256,6 +261,85 @@ public void testClearSnapshotSegments() throws Exception { processor.closeAsync().get(5, TimeUnit.SECONDS); } + @Test + public void testTxnSegmentStats() throws Exception { + // Set up test environment + String namespace = TENANT + "/testTxnSegmentStats"; + String topic = "persistent://" + namespace + "/testTxnSegmentStats"; + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + topic.length() + SEGMENT_SIZE * 3); + + // Create necessary resources + Transactions transactions = admin.transactions(); + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + + // Prepare topic, producer, and consumer + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("my-sub").subscribe(); + + // Record the start time of the test + long testStartTime = System.currentTimeMillis(); + + Transaction transaction = null; + // Send messages with transactions and abort them + for (int i = 0; i < SEGMENT_SIZE; i++) { + transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS).build().get(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + } + + Transaction txn = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.HOURS).build().get(); + producer.newMessage(txn).send(); + txn.abort().get(); + + // Get the transaction buffer stats without segment stats + TransactionBufferStats statsWithoutSegmentStats = transactions + .getTransactionBufferStats(topic, false, false); + assertNotNull(statsWithoutSegmentStats); + assertNotNull(statsWithoutSegmentStats.segmentsStats); + assertNull(statsWithoutSegmentStats.segmentsStats.segmentStats); + assertEquals(statsWithoutSegmentStats.snapshotType, AbortedTxnProcessor.SnapshotType.Segment.toString()); + + // Verify the segment stats + assertEquals(statsWithoutSegmentStats.segmentsStats.segmentsSize, 1L); + assertEquals(statsWithoutSegmentStats.segmentsStats.unsealedAbortTxnIDSize, 1L); + assertEquals(statsWithoutSegmentStats.segmentsStats.currentSegmentCapacity, SEGMENT_SIZE); + assertEquals(statsWithoutSegmentStats.totalAbortedTransactions, SEGMENT_SIZE + 1); + assertTrue(statsWithoutSegmentStats.segmentsStats.lastTookSnapshotSegmentTimestamp >= testStartTime); + + // Get the transaction buffer stats with segment stats + TransactionBufferStats statsWithSegmentStats = transactions + .getTransactionBufferStats(topic, false, true); + assertNotNull(statsWithSegmentStats); + assertNotNull(statsWithSegmentStats.segmentsStats.segmentStats); + + // Verify if the segment stats are present when requested + assertEquals(statsWithSegmentStats.segmentsStats.segmentStats.size(), 1); + assertEquals(statsWithSegmentStats.segmentsStats.segmentStats.get(0).lastTxnID, + transaction.getTxnID().toString()); + + // Verify multiple segments + for (int i = 0; i < SEGMENT_SIZE * 3; i++) { + transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS).build().get(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + } + statsWithSegmentStats = transactions.getTransactionBufferStats(topic, false, true); + + // Verify the segment stats + assertEquals(statsWithSegmentStats.segmentsStats.segmentsSize, 4L); + assertEquals(statsWithSegmentStats.segmentsStats.unsealedAbortTxnIDSize, 1L); + assertEquals(statsWithSegmentStats.totalAbortedTransactions, SEGMENT_SIZE * 4 + 1); + + // Reset the configuration + this.pulsarService.getConfig() + .setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + SEGMENT_SIZE * 3); + } + private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { SystemTopicClient.Reader reader = pulsarService.getTransactionBufferSnapshotServiceFactory() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 4e50401fc11eb..cf389824794e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -743,7 +743,7 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ TransactionBuffer buffer2 = new TopicTransactionBuffer(persistentTopic); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> - assertEquals(buffer2.getStats(false).state, "Ready")); + assertEquals(buffer2.getStats(false, false).state, "Ready")); managedCursors.removeCursor("transaction-buffer-sub"); doAnswer(invocation -> { @@ -755,7 +755,7 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ managedCursors.add(managedCursor, managedCursor.getMarkDeletedPosition()); TransactionBuffer buffer3 = new TopicTransactionBuffer(persistentTopic); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> - assertEquals(buffer3.getStats(false).state, "Ready")); + assertEquals(buffer3.getStats(false, false).state, "Ready")); persistentTopic.getInternalStats(false).thenAccept(internalStats -> { assertTrue(internalStats.cursors.isEmpty()); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index cd0c089ad41be..c0300c63b3587 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -58,6 +58,7 @@ public abstract class TransactionTestBase extends TestRetrySupport { public static final String CLUSTER_NAME = "test"; @Setter + @Getter private int brokerCount = 3; @Getter private final List serviceConfigurationList = new ArrayList<>(); @@ -116,7 +117,7 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int createTransactionCoordinatorAssign(numPartitionsOfTC); admin.tenants().createTenant(TENANT, new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); + admin.namespaces().createNamespace(NAMESPACE1, 4); if (topic != null) { if (numPartitions == 0) { admin.topics().createNonPartitionedTopic(topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java index 55d115905a35d..7493b25ac1d90 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java @@ -235,10 +235,10 @@ private void checkTopicTransactionBufferState(boolean clientEnableTransaction, Awaitility.await().until(() -> { if (clientEnableTransaction) { // recover success, client enable transaction will change to Ready State - return topicTransactionBuffer.getStats(false).state.equals(Ready.name()); + return topicTransactionBuffer.getStats(false, false).state.equals(Ready.name()); } else { // recover success, client disable transaction will change to NoSnapshot State - return topicTransactionBuffer.getStats(false).state.equals(NoSnapshot.name()); + return topicTransactionBuffer.getStats(false, false).state.equals(NoSnapshot.name()); } }); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java index 57adf263a574f..b0504bee744c8 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java @@ -23,6 +23,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -139,10 +140,24 @@ TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID, String * Get transaction buffer stats. * * @param topic the topic of getting transaction buffer stats - * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction pending ack. + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction pending ack. + * @param segmentStats Whether to get segment statistics. * @return the future stats of transaction buffer in topic. */ - CompletableFuture getTransactionBufferStatsAsync(String topic, boolean lowWaterMarks); + CompletableFuture getTransactionBufferStatsAsync(String topic, boolean lowWaterMarks, + boolean segmentStats); + + /** + * Get transaction buffer stats. + * + * @param topic the topic of getting transaction buffer stats + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction pending ack. + * @return the future stats of transaction buffer in topic. + */ + default CompletableFuture getTransactionBufferStatsAsync(String topic, + boolean lowWaterMarks) { + return getTransactionBufferStatsAsync(topic, lowWaterMarks, false); + } /** * Get transaction buffer stats. @@ -151,17 +166,31 @@ TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID, String * @return the future stats of transaction buffer in topic. */ default CompletableFuture getTransactionBufferStatsAsync(String topic) { - return getTransactionBufferStatsAsync(topic, false); + return getTransactionBufferStatsAsync(topic, false, false); } /** * Get transaction buffer stats. * * @param topic the topic of getting transaction buffer stats - * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction buffer. + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction buffer. + * @param segmentStats Whether to get segment statistics. * @return the stats of transaction buffer in topic. */ - TransactionBufferStats getTransactionBufferStats(String topic, boolean lowWaterMarks) throws PulsarAdminException; + TransactionBufferStats getTransactionBufferStats(String topic, boolean lowWaterMarks, + boolean segmentStats) throws PulsarAdminException; + + /** + * Get transaction buffer stats. + * + * @param topic the topic of getting transaction buffer stats + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction buffer. + * @return the stats of transaction buffer in topic. + */ + default TransactionBufferStats getTransactionBufferStats(String topic, + boolean lowWaterMarks) throws PulsarAdminException { + return getTransactionBufferStats(topic, lowWaterMarks, false); + } /** * Get transaction buffer stats. @@ -170,7 +199,7 @@ default CompletableFuture getTransactionBufferStatsAsync * @return the stats of transaction buffer in topic. */ default TransactionBufferStats getTransactionBufferStats(String topic) throws PulsarAdminException { - return getTransactionBufferStats(topic, false); + return getTransactionBufferStats(topic, false, false); } /** @@ -309,6 +338,28 @@ CompletableFuture getPendingAckInternalStats TransactionPendingAckInternalStats getPendingAckInternalStats(String topic, String subName, boolean metadata) throws PulsarAdminException; + /** + * Get transaction buffer internal stats asynchronously. + * + * @param topic the topic to get transaction buffer internal stats from + * @param metadata whether to obtain ledger metadata + * + * @return the future internal stats of transaction buffer + */ + CompletableFuture getTransactionBufferInternalStatsAsync(String topic, + boolean metadata); + + /** + * Get transaction buffer internal stats. + * + * @param topic the topic to get transaction buffer internal stats from + * @param metadata whether to obtain ledger metadata + * + * @return the internal stats of transaction buffer + */ + TransactionBufferInternalStats getTransactionBufferInternalStats(String topic, + boolean metadata) throws PulsarAdminException; + /** * Sets the scale of the transaction coordinators. * And currently, we can only support scale-up. diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java new file mode 100644 index 0000000000000..007f4f4d63245 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +public class SegmentStats { + public String lastTxnID; + public String persistentPosition; + + public SegmentStats(String lastTxnID, String persistentPosition) { + this.lastTxnID = lastTxnID; + this.persistentPosition = persistentPosition; + } + + public SegmentStats() { + } +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java new file mode 100644 index 0000000000000..46422c0b67b58 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +import java.util.List; + +public class SegmentsStats { + // The current number of the snapshot segments. + public long segmentsSize; + + // The capacity of snapshot segment calculated by the current config (transactionBufferSnapshotSegmentSize) + public long currentSegmentCapacity; + + // The latest aborted txn IDs which number less than currentSegmentCapacity + public long unsealedAbortTxnIDSize; + + // A list of individual segment stats + public List segmentStats; + /** The last snapshot segment timestamps of this transaction buffer. */ + public long lastTookSnapshotSegmentTimestamp; +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java new file mode 100644 index 0000000000000..7ce95375e7299 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +public class SnapshotSystemTopicInternalStats { + // The managed ledger name for the snapshot segment topic or index topic. + public String managedLedgerName; + + // The managed ledger internal stats for the snapshot segment topic or index topic. + public ManagedLedgerInternalStats managedLedgerInternalStats; +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java new file mode 100644 index 0000000000000..b4c9e096d0a11 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +public class TransactionBufferInternalStats { + // The type of snapshot being used: either "Single" or "Segment" + public String snapshotType; + + // If snapshotType is "Single", this field will provide the statistics of single snapshot log. + public SnapshotSystemTopicInternalStats singleSnapshotSystemTopicInternalStats; + + // If snapshotType is "Segment", this field will provide the statistics of snapshot segment topic. + public SnapshotSystemTopicInternalStats segmentInternalStats; + + // If snapshotType is "Segment", this field will provide the statistics of snapshot segment index topic. + public SnapshotSystemTopicInternalStats segmentIndexInternalStats; +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java index 73d66b8c230bb..1dffa0dd61481 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java @@ -44,4 +44,13 @@ public class TransactionBufferStats { public long recoverStartTime; //End timestamp of transaction buffer recovery. 0L means no startup. public long recoverEndTime; + + // The total number of aborted transactions. + public long totalAbortedTransactions; + + // The type of snapshot being used: either "Single" or "Segment" + public String snapshotType; + + // If snapshotType is "Segment", this field will provide additional segment-related statistics + public SegmentsStats segmentsStats; } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java index 5693ebc8f60aa..2d1dd408ef6c9 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -132,17 +133,20 @@ public TransactionMetadata getTransactionMetadata(TxnID txnID) throws PulsarAdmi @Override public CompletableFuture getTransactionBufferStatsAsync(String topic, - boolean lowWaterMarks) { + boolean lowWaterMarks, + boolean segmentStats) { WebTarget path = adminV3Transactions.path("transactionBufferStats"); path = path.path(TopicName.get(topic).getRestPath(false)); path = path.queryParam("lowWaterMarks", lowWaterMarks); + path = path.queryParam("segmentStats", segmentStats); return asyncGetRequest(path, new FutureCallback(){}); } @Override public TransactionBufferStats getTransactionBufferStats(String topic, - boolean lowWaterMarks) throws PulsarAdminException { - return sync(() -> getTransactionBufferStatsAsync(topic, lowWaterMarks)); + boolean lowWaterMarks, + boolean segmentStats) throws PulsarAdminException { + return sync(() -> getTransactionBufferStatsAsync(topic, lowWaterMarks, segmentStats)); } @Override @@ -227,6 +231,22 @@ public TransactionPendingAckInternalStats getPendingAckInternalStats(String topi return sync(() -> getPendingAckInternalStatsAsync(topic, subName, metadata)); } + @Override + public CompletableFuture getTransactionBufferInternalStatsAsync(String topic, + boolean metadata) { + TopicName tn = TopicName.get(topic); + WebTarget path = adminV3Transactions.path("transactionBufferInternalStats"); + path = path.path(tn.getRestPath(false)); + path = path.queryParam("metadata", metadata); + return asyncGetRequest(path, new FutureCallback(){}); + } + + @Override + public TransactionBufferInternalStats getTransactionBufferInternalStats(String topic, boolean metadata) + throws PulsarAdminException { + return sync(() -> getTransactionBufferInternalStatsAsync(topic, metadata)); + } + @Override public void scaleTransactionCoordinators(int replicas) throws PulsarAdminException { sync(() -> scaleTransactionCoordinatorsAsync(replicas)); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 8509d037cbabc..a722abe19df81 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -2377,7 +2377,7 @@ void transactions() throws Exception { cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("transaction-buffer-stats -t test -l")); - verify(transactions).getTransactionBufferStats("test", true); + verify(transactions).getTransactionBufferStats("test", true, false); cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("pending-ack-stats -t test -s test -l")); @@ -2387,6 +2387,10 @@ void transactions() throws Exception { cmdTransactions.run(split("pending-ack-internal-stats -t test -s test")); verify(transactions).getPendingAckInternalStats("test", "test", false); + cmdTransactions = new CmdTransactions(() -> admin); + cmdTransactions.run(split("buffer-snapshot-internal-stats -t test")); + verify(transactions).getTransactionBufferInternalStats("test", false); + cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("scale-transactionCoordinators -r 3")); verify(transactions).scaleTransactionCoordinators(3); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java index 08ffba1451f23..b999e30b108f2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java @@ -56,9 +56,14 @@ private class GetTransactionBufferStats extends CliCommand { description = "Whether to get information about lowWaterMarks stored in transaction buffer.") private boolean lowWaterMark; + @Parameter(names = {"-s", "--segment-stats"}, + description = "Whether to get segment statistics.") + private boolean segmentStats = false; + @Override void run() throws Exception { - print(getAdmin().transactions().getTransactionBufferStats(topic, lowWaterMark)); + // Assuming getTransactionBufferStats method signature has been updated to accept the new parameter + print(getAdmin().transactions().getTransactionBufferStats(topic, lowWaterMark, segmentStats)); } } @@ -188,6 +193,20 @@ void run() throws Exception { } } + @Parameters(commandDescription = "Get transaction buffer internal stats") + private class GetTransactionBufferInternalStats extends CliCommand { + @Parameter(names = {"-t", "--topic"}, description = "Topic name", required = true) + private String topic; + + @Parameter(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") + private boolean metadata = false; + + @Override + void run() throws Exception { + print(getAdmin().transactions().getTransactionBufferInternalStats(topic, metadata)); + } + } + @Parameters(commandDescription = "Update the scale of transaction coordinators") private class ScaleTransactionCoordinators extends CliCommand { @Parameter(names = { "-r", "--replicas" }, description = "The scale of the transaction coordinators") @@ -242,6 +261,7 @@ public CmdTransactions(Supplier admin) { super("transactions", admin); jcommander.addCommand("coordinator-internal-stats", new GetCoordinatorInternalStats()); jcommander.addCommand("pending-ack-internal-stats", new GetPendingAckInternalStats()); + jcommander.addCommand("buffer-snapshot-internal-stats", new GetTransactionBufferInternalStats()); jcommander.addCommand("coordinator-stats", new GetCoordinatorStats()); jcommander.addCommand("transaction-buffer-stats", new GetTransactionBufferStats()); jcommander.addCommand("pending-ack-stats", new GetPendingAckStats()); From 54f0ca1c769638000fbeb4d73a565216c0992c32 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Sat, 22 Jul 2023 21:40:47 +0200 Subject: [PATCH 11/29] [cleanup][io] Cleanup Kafka connector (#20721) --- .../apache/pulsar/functions/api/Record.java | 2 +- .../pulsar/io/kafka/AvroSchemaCache.java | 4 +- .../io/kafka/ByteBufferSchemaWrapper.java | 2 +- .../pulsar/io/kafka/BytesWithKafkaSchema.java | 4 +- .../pulsar/io/kafka/KafkaAbstractSink.java | 2 +- .../pulsar/io/kafka/KafkaAbstractSource.java | 25 +-- .../pulsar/io/kafka/KafkaBytesSink.java | 2 +- .../pulsar/io/kafka/KafkaBytesSource.java | 36 ++-- .../pulsar/io/kafka/KafkaSinkConfig.java | 7 - .../pulsar/io/kafka/KafkaSourceConfig.java | 7 - .../pulsar/io/kafka/KafkaStringSource.java | 6 +- .../io/kafka/ByteBufferSchemaWrapperTest.java | 8 +- .../pulsar/io/kafka/KafkaBytesSourceTest.java | 201 +++++++++--------- .../io/kafka/sink/KafkaAbstractSinkTest.java | 4 +- .../kafka/source/KafkaAbstractSourceTest.java | 19 +- 15 files changed, 158 insertions(+), 171 deletions(-) diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java index ea6987e5f0391..0487b3d02b3a1 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java @@ -27,7 +27,7 @@ import org.apache.pulsar.common.classification.InterfaceStability; /** - * Pulsar Connect's Record interface. Record encapsulates the information about a record being read from a Source. + * Pulsar IO's Record interface. Record encapsulates the information about a record being read from a Source. */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java index f0ad79549eb13..4e3abe245beb9 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java @@ -38,9 +38,9 @@ final class AvroSchemaCache { private final LoadingCache> cache = CacheBuilder .newBuilder() .maximumSize(100) - .build(new CacheLoader>() { + .build(new CacheLoader<>() { @Override - public Schema load(Integer schemaId) throws Exception { + public Schema load(Integer schemaId) { return fetchSchema(schemaId); } }); diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java index fb4683e511dd6..aefe62e38570c 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java @@ -29,7 +29,7 @@ class ByteBufferSchemaWrapper implements Schema { private final Supplier original; - public ByteBufferSchemaWrapper(Schema original) { + public ByteBufferSchemaWrapper(Schema original) { this(original::getSchemaInfo); } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java index f0429c92eedfd..585cd927c30cd 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java @@ -26,6 +26,6 @@ */ @Value public class BytesWithKafkaSchema { - private final ByteBuffer value; - private final int schemaId; + ByteBuffer value; + int schemaId; } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java index 8fbd6c8186110..5ceea4dec8dca 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java @@ -44,7 +44,7 @@ public abstract class KafkaAbstractSink implements Sink { private Producer producer; - private Properties props = new Properties(); + private final Properties props = new Properties(); private KafkaSinkConfig kafkaSinkConfig; @Override diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java index 3d4612c039f36..f8539518851aa 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java @@ -166,7 +166,7 @@ public void start() { CompletableFuture[] futures = new CompletableFuture[consumerRecords.count()]; int index = 0; for (ConsumerRecord consumerRecord : consumerRecords) { - KafkaRecord record = buildRecord(consumerRecord); + KafkaRecord record = buildRecord(consumerRecord); if (LOG.isDebugEnabled()) { LOG.debug("Write record {} {} {}", record.getKey(), record.getValue(), record.getSchema()); } @@ -190,7 +190,7 @@ public void start() { runnerThread.start(); } - public abstract KafkaRecord buildRecord(ConsumerRecord consumerRecord); + public abstract KafkaRecord buildRecord(ConsumerRecord consumerRecord); protected Map copyKafkaHeaders(ConsumerRecord consumerRecord) { if (!kafkaSourceConfig.isCopyHeadersEnabled()) { @@ -208,7 +208,7 @@ protected Map copyKafkaHeaders(ConsumerRecord co @Slf4j protected static class KafkaRecord implements Record { - private final ConsumerRecord record; + private final ConsumerRecord record; private final V value; private final Schema schema; private final Map properties; @@ -216,7 +216,7 @@ protected static class KafkaRecord implements Record { @Getter private final CompletableFuture completableFuture = new CompletableFuture<>(); - public KafkaRecord(ConsumerRecord record, V value, Schema schema, + public KafkaRecord(ConsumerRecord record, V value, Schema schema, Map properties) { this.record = record; this.value = value; @@ -240,7 +240,7 @@ public Optional getRecordSequence() { @Override public Optional getKey() { - return Optional.ofNullable(record.key()); + return Optional.ofNullable(record.key() instanceof String ? (String) record.key() : null); } @Override @@ -263,13 +263,14 @@ public Map getProperties(){ return properties; } } - protected static class KeyValueKafkaRecord extends KafkaRecord implements KVRecord { - private final Schema keySchema; - private final Schema valueSchema; + protected static class KeyValueKafkaRecord extends KafkaRecord implements KVRecord { - public KeyValueKafkaRecord(ConsumerRecord record, KeyValue value, - Schema keySchema, Schema valueSchema, + private final Schema keySchema; + private final Schema valueSchema; + + public KeyValueKafkaRecord(ConsumerRecord record, KeyValue value, + Schema keySchema, Schema valueSchema, Map properties) { super(record, value, null, properties); this.keySchema = keySchema; @@ -277,12 +278,12 @@ public KeyValueKafkaRecord(ConsumerRecord record, KeyValue value, } @Override - public Schema getKeySchema() { + public Schema getKeySchema() { return keySchema; } @Override - public Schema getValueSchema() { + public Schema getValueSchema() { return valueSchema; } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java index 2a200531b67cb..0ed2385699511 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java @@ -29,7 +29,7 @@ import org.apache.pulsar.io.core.annotations.IOType; /** - * Kafka sink should treats incoming messages as pure bytes. So we don't + * Kafka sink should treat incoming messages as pure bytes. So we don't * apply schema into it. */ @Connector( diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java index 4e35d98e0bb71..51408f0451977 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java @@ -70,8 +70,8 @@ public class KafkaBytesSource extends KafkaAbstractSource { private AvroSchemaCache schemaCache; - private Schema keySchema; - private Schema valueSchema; + private Schema keySchema; + private Schema valueSchema; private boolean produceKeyValue; @Override @@ -93,11 +93,11 @@ protected Properties beforeCreateConsumer(Properties props) { } if (keySchema.getSchemaInfo().getType() != SchemaType.STRING) { - // if the Key is a String we can use native Pulsar Key - // otherwise we use KeyValue schema - // that allows you to set a schema for the Key and a schema for the Value. - // using SEPARATED encoding the key is saved into the binary key - // so it is used for routing and for compaction + // If the Key is a String we can use native Pulsar Key. + // Otherwise, we use KeyValue schema. + // That allows you to set a schema for the Key and a schema for the Value. + // Using SEPARATED encoding the key is saved into the binary key, + // so it is used for routing and for compaction. produceKeyValue = true; } @@ -114,13 +114,13 @@ private void initSchemaCache(Properties props) { } @Override - public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { + public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { if (produceKeyValue) { - Object key = extractSimpleValue(consumerRecord.key()); - Object value = extractSimpleValue(consumerRecord.value()); - Schema currentKeySchema = getSchemaFromObject(consumerRecord.key(), keySchema); - Schema currentValueSchema = getSchemaFromObject(consumerRecord.value(), valueSchema); - return new KeyValueKafkaRecord(consumerRecord, + ByteBuffer key = extractSimpleValue(consumerRecord.key()); + ByteBuffer value = extractSimpleValue(consumerRecord.value()); + Schema currentKeySchema = getSchemaFromObject(consumerRecord.key(), keySchema); + Schema currentValueSchema = getSchemaFromObject(consumerRecord.value(), valueSchema); + return new KeyValueKafkaRecord(consumerRecord, new KeyValue<>(key, value), currentKeySchema, currentValueSchema, @@ -128,7 +128,7 @@ public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { } else { Object value = consumerRecord.value(); - return new KafkaRecord(consumerRecord, + return new KafkaRecord<>(consumerRecord, extractSimpleValue(value), getSchemaFromObject(value, valueSchema), copyKafkaHeaders(consumerRecord)); @@ -152,7 +152,7 @@ private static ByteBuffer extractSimpleValue(Object value) { } } - private Schema getSchemaFromObject(Object value, Schema fallback) { + private Schema getSchemaFromObject(Object value, Schema fallback) { if (value instanceof BytesWithKafkaSchema) { // this is a Struct with schema downloaded by the schema registry // the schema may be different from record to record @@ -179,7 +179,7 @@ private static Schema getSchemaFromDeserializerAndAdaptConfiguration result = Schema.BYTEBUFFER; } else if (StringDeserializer.class.getName().equals(kafkaDeserializerClass)) { if (isKey) { - // for the key we use the String value and we want StringDeserializer + // for the key we use the String value, and we want StringDeserializer props.put(key, kafkaDeserializerClass); } result = Schema.STRING; @@ -206,11 +206,11 @@ private static Schema getSchemaFromDeserializerAndAdaptConfiguration return new ByteBufferSchemaWrapper(result); } - Schema getKeySchema() { + Schema getKeySchema() { return keySchema; } - Schema getValueSchema() { + Schema getValueSchema() { return valueSchema; } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java index dbbf1a7b5e2a0..755b2c89c8f20 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java @@ -43,43 +43,36 @@ public class KafkaSinkConfig implements Serializable { private String bootstrapServers; @FieldDoc( - required = false, defaultValue = "", help = "Protocol used to communicate with Kafka brokers.") private String securityProtocol; @FieldDoc( - required = false, defaultValue = "", help = "SASL mechanism used for Kafka client connections.") private String saslMechanism; @FieldDoc( - required = false, defaultValue = "", help = "JAAS login context parameters for SASL connections in the format used by JAAS configuration files.") private String saslJaasConfig; @FieldDoc( - required = false, defaultValue = "", help = "The list of protocols enabled for SSL connections.") private String sslEnabledProtocols; @FieldDoc( - required = false, defaultValue = "", help = "The endpoint identification algorithm to validate server hostname using server certificate.") private String sslEndpointIdentificationAlgorithm; @FieldDoc( - required = false, defaultValue = "", help = "The location of the trust store file.") private String sslTruststoreLocation; @FieldDoc( - required = false, defaultValue = "", help = "The password for the trust store file.") private String sslTruststorePassword; diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java index ad2e121d26abf..5de60d2a028c8 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java @@ -44,43 +44,36 @@ public class KafkaSourceConfig implements Serializable { private String bootstrapServers; @FieldDoc( - required = false, defaultValue = "", help = "Protocol used to communicate with Kafka brokers.") private String securityProtocol; @FieldDoc( - required = false, defaultValue = "", help = "SASL mechanism used for Kafka client connections.") private String saslMechanism; @FieldDoc( - required = false, defaultValue = "", help = "JAAS login context parameters for SASL connections in the format used by JAAS configuration files.") private String saslJaasConfig; @FieldDoc( - required = false, defaultValue = "", help = "The list of protocols enabled for SSL connections.") private String sslEnabledProtocols; @FieldDoc( - required = false, defaultValue = "", help = "The endpoint identification algorithm to validate server hostname using server certificate.") private String sslEndpointIdentificationAlgorithm; @FieldDoc( - required = false, defaultValue = "", help = "The location of the trust store file.") private String sslTruststoreLocation; @FieldDoc( - required = false, defaultValue = "", help = "The password for the trust store file.") private String sslTruststorePassword; diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java index 58df1838b2eb1..0c96528cc9349 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java @@ -27,14 +27,12 @@ */ public class KafkaStringSource extends KafkaAbstractSource { - @Override - public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { - KafkaRecord record = new KafkaRecord(consumerRecord, + public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { + return new KafkaRecord<>(consumerRecord, new String((byte[]) consumerRecord.value(), StandardCharsets.UTF_8), Schema.STRING, copyKafkaHeaders(consumerRecord)); - return record; } } diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java index e7f108bf60146..eff4b294851c3 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java @@ -32,7 +32,7 @@ public class ByteBufferSchemaWrapperTest { @Test - public void testGetBytesNoCopy() throws Exception { + public void testGetBytesNoCopy() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray); assertEquals(0, wrapped.arrayOffset()); @@ -41,7 +41,7 @@ public void testGetBytesNoCopy() throws Exception { } @Test - public void testGetBytesOffsetZeroDifferentLen() throws Exception { + public void testGetBytesOffsetZeroDifferentLen() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray, 1, 2); assertEquals(0, wrapped.arrayOffset()); @@ -52,7 +52,7 @@ public void testGetBytesOffsetZeroDifferentLen() throws Exception { } @Test - public void testGetBytesOffsetNonZero() throws Exception { + public void testGetBytesOffsetNonZero() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray); wrapped.position(1); @@ -66,7 +66,7 @@ public void testGetBytesOffsetNonZero() throws Exception { } @Test - public void testGetBytesOffsetZero() throws Exception { + public void testGetBytesOffsetZero() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray, 0, 2); assertEquals(0, wrapped.arrayOffset()); diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java index d370d51cc23c1..401bd64c36f84 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java @@ -44,6 +44,8 @@ import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.io.core.SourceContext; +import org.apache.pulsar.io.kafka.KafkaAbstractSource.KafkaRecord; +import org.apache.pulsar.io.kafka.KafkaAbstractSource.KeyValueKafkaRecord; import org.bouncycastle.util.encoders.Base64; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -88,24 +90,25 @@ public void testNoKeyValueSchema() throws Exception { } - private void validateSchemaNoKeyValue(String keyDeserializationClass, Schema expectedKeySchema, - String valueDeserializationClass, Schema expectedValueSchema) throws Exception { - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", valueDeserializationClass); - config.put("keyDeserializationClass", keyDeserializationClass); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - assertFalse(source.isProduceKeyValue()); - Schema keySchema = source.getKeySchema(); - Schema valueSchema = source.getValueSchema(); - assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); - assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); + private void validateSchemaNoKeyValue(String keyDeserializationClass, Schema expectedKeySchema, + String valueDeserializationClass, Schema expectedValueSchema) throws Exception { + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", valueDeserializationClass); + config.put("keyDeserializationClass", keyDeserializationClass); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + assertFalse(source.isProduceKeyValue()); + Schema keySchema = source.getKeySchema(); + Schema valueSchema = source.getValueSchema(); + assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); + assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); + } } @Test @@ -120,96 +123,98 @@ public void testKeyValueSchema() throws Exception { public void testCopyKafkaHeadersEnabled() throws Exception { ByteBuffer key = ByteBuffer.wrap(new IntegerSerializer().serialize("test", 10)); ByteBuffer value = ByteBuffer.wrap(new StringSerializer().serialize("test", "test")); - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("copyHeadersEnabled", true); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); - config.put("keyDeserializationClass", StringDeserializer.class.getName()); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - ConsumerRecord record = new ConsumerRecord("test", 88, 99, key, value); - record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); - record.headers().add("k2", new byte[]{0xF}); - - Map props = source.copyKafkaHeaders(record); - assertEquals(props.size(), 5); - assertTrue(props.containsKey("__kafka_topic")); - assertTrue(props.containsKey("__kafka_partition")); - assertTrue(props.containsKey("__kafka_offset")); - assertTrue(props.containsKey("k1")); - assertTrue(props.containsKey("k2")); - - assertEquals(props.get("__kafka_topic"), "test"); - assertEquals(props.get("__kafka_partition"), "88"); - assertEquals(props.get("__kafka_offset"), "99"); - assertEquals(Base64.decode(props.get("k1")), "v1".getBytes(StandardCharsets.UTF_8)); - assertEquals(Base64.decode(props.get("k2")), new byte[]{0xF}); + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("copyHeadersEnabled", true); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); + config.put("keyDeserializationClass", StringDeserializer.class.getName()); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + ConsumerRecord record = new ConsumerRecord<>("test", 88, 99, key, value); + record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); + record.headers().add("k2", new byte[]{0xF}); + + Map props = source.copyKafkaHeaders(record); + assertEquals(props.size(), 5); + assertTrue(props.containsKey("__kafka_topic")); + assertTrue(props.containsKey("__kafka_partition")); + assertTrue(props.containsKey("__kafka_offset")); + assertTrue(props.containsKey("k1")); + assertTrue(props.containsKey("k2")); + + assertEquals(props.get("__kafka_topic"), "test"); + assertEquals(props.get("__kafka_partition"), "88"); + assertEquals(props.get("__kafka_offset"), "99"); + assertEquals(Base64.decode(props.get("k1")), "v1".getBytes(StandardCharsets.UTF_8)); + assertEquals(Base64.decode(props.get("k2")), new byte[]{0xF}); + } } @Test public void testCopyKafkaHeadersDisabled() throws Exception { ByteBuffer key = ByteBuffer.wrap(new IntegerSerializer().serialize("test", 10)); ByteBuffer value = ByteBuffer.wrap(new StringSerializer().serialize("test", "test")); - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); - config.put("keyDeserializationClass", StringDeserializer.class.getName()); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - ConsumerRecord record = new ConsumerRecord("test", 88, 99, key, value); - record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); - record.headers().add("k2", new byte[]{0xF}); - - Map props = source.copyKafkaHeaders(record); - assertTrue(props.isEmpty()); + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); + config.put("keyDeserializationClass", StringDeserializer.class.getName()); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + ConsumerRecord record = new ConsumerRecord<>("test", 88, 99, key, value); + record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); + record.headers().add("k2", new byte[]{0xF}); + + Map props = source.copyKafkaHeaders(record); + assertTrue(props.isEmpty()); + } } - private void validateSchemaKeyValue(String keyDeserializationClass, Schema expectedKeySchema, - String valueDeserializationClass, Schema expectedValueSchema, + private void validateSchemaKeyValue(String keyDeserializationClass, Schema expectedKeySchema, + String valueDeserializationClass, Schema expectedValueSchema, ByteBuffer key, ByteBuffer value) throws Exception { - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", valueDeserializationClass); - config.put("keyDeserializationClass", keyDeserializationClass); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - assertTrue(source.isProduceKeyValue()); - Schema keySchema = source.getKeySchema(); - Schema valueSchema = source.getValueSchema(); - assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); - assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); - - KafkaAbstractSource.KafkaRecord record = source.buildRecord(new ConsumerRecord("test", 0, 0, key, value)); - assertThat(record, instanceOf(KafkaAbstractSource.KeyValueKafkaRecord.class)); - KafkaAbstractSource.KeyValueKafkaRecord kvRecord = (KafkaAbstractSource.KeyValueKafkaRecord) record; - assertSame(keySchema, kvRecord.getKeySchema()); - assertSame(valueSchema, kvRecord.getValueSchema()); - assertEquals(KeyValueEncodingType.SEPARATED, kvRecord.getKeyValueEncodingType()); - KeyValue kvValue = (KeyValue) kvRecord.getValue(); - log.info("key {}", Arrays.toString(toArray(key))); - log.info("value {}", Arrays.toString(toArray(value))); - - log.info("key {}", Arrays.toString(toArray((ByteBuffer) kvValue.getKey()))); - log.info("value {}", Arrays.toString(toArray((ByteBuffer) kvValue.getValue()))); - - assertEquals(ByteBuffer.wrap(toArray(key)).compareTo((ByteBuffer) kvValue.getKey()), 0); - assertEquals(ByteBuffer.wrap(toArray(value)).compareTo((ByteBuffer) kvValue.getValue()), 0); + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", valueDeserializationClass); + config.put("keyDeserializationClass", keyDeserializationClass); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + assertTrue(source.isProduceKeyValue()); + Schema keySchema = source.getKeySchema(); + Schema valueSchema = source.getValueSchema(); + assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); + assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); + + KafkaRecord record = source.buildRecord(new ConsumerRecord<>("test", 0, 0, key, value)); + assertThat(record, instanceOf(KeyValueKafkaRecord.class)); + KeyValueKafkaRecord kvRecord = (KeyValueKafkaRecord) record; + assertSame(keySchema, kvRecord.getKeySchema()); + assertSame(valueSchema, kvRecord.getValueSchema()); + assertEquals(KeyValueEncodingType.SEPARATED, kvRecord.getKeyValueEncodingType()); + KeyValue kvValue = (KeyValue) kvRecord.getValue(); + log.info("key {}", Arrays.toString(toArray(key))); + log.info("value {}", Arrays.toString(toArray(value))); + log.info("key {}", Arrays.toString(toArray(kvValue.getKey()))); + log.info("value {}", Arrays.toString(toArray(kvValue.getValue()))); + + assertEquals(ByteBuffer.wrap(toArray(key)).compareTo(kvValue.getKey()), 0); + assertEquals(ByteBuffer.wrap(toArray(value)).compareTo(kvValue.getValue()), 0); + } } private static byte[] toArray(ByteBuffer b) { diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java index ec9ee4a957d78..d59cdb1d9b63d 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java @@ -47,7 +47,7 @@ public class KafkaAbstractSinkTest { private static class DummySink extends KafkaAbstractSink { @Override - public KeyValue extractKeyValue(Record record) { + public KeyValue extractKeyValue(Record record) { return new KeyValue<>(record.getKey().orElse(null), record.getValue()); } } @@ -74,7 +74,7 @@ private static void expectThrows(Class expectedType, St @Test public void testInvalidConfigWillThrownException() throws Exception { - KafkaAbstractSink sink = new DummySink(); + KafkaAbstractSink sink = new DummySink(); Map config = new HashMap<>(); SinkContext sc = new SinkContext() { @Override diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java index 6911ec2a6bfa3..9e0fef87a2592 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java @@ -18,10 +18,8 @@ */ package org.apache.pulsar.io.kafka.source; - import com.google.common.collect.ImmutableMap; import java.time.Duration; -import java.util.Collection; import java.util.Collections; import java.lang.reflect.Field; import org.apache.kafka.clients.consumer.Consumer; @@ -57,18 +55,17 @@ public class KafkaAbstractSourceTest { private static class DummySource extends KafkaAbstractSource { @Override - public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { - KafkaRecord record = new KafkaRecord(consumerRecord, + public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { + return new KafkaRecord<>(consumerRecord, new String((byte[]) consumerRecord.value(), StandardCharsets.UTF_8), Schema.STRING, Collections.emptyMap()); - return record; } } @Test public void testInvalidConfigWillThrownException() throws Exception { - KafkaAbstractSource source = new DummySource(); + KafkaAbstractSource source = new DummySource(); SourceContext ctx = mock(SourceContext.class); Map config = new HashMap<>(); Assert.ThrowingRunnable openAndClose = ()->{ @@ -160,7 +157,7 @@ public final void loadFromSaslYamlFileTest() throws IOException { @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Subscribe exception") public final void throwExceptionBySubscribe() throws Exception { - KafkaAbstractSource source = new DummySource(); + KafkaAbstractSource source = new DummySource(); KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); kafkaSourceConfig.setTopic("test-topic"); @@ -168,9 +165,9 @@ public final void throwExceptionBySubscribe() throws Exception { kafkaSourceConfigField.setAccessible(true); kafkaSourceConfigField.set(source, kafkaSourceConfig); - Consumer consumer = mock(Consumer.class); + Consumer consumer = mock(Consumer.class); Mockito.doThrow(new RuntimeException("Subscribe exception")).when(consumer) - .subscribe(Mockito.any(Collection.class)); + .subscribe(Mockito.anyCollection()); Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); consumerField.setAccessible(true); @@ -181,7 +178,7 @@ public final void throwExceptionBySubscribe() throws Exception { @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Pool exception") public final void throwExceptionByPoll() throws Exception { - KafkaAbstractSource source = new DummySource(); + KafkaAbstractSource source = new DummySource(); KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); kafkaSourceConfig.setTopic("test-topic"); @@ -189,7 +186,7 @@ public final void throwExceptionByPoll() throws Exception { kafkaSourceConfigField.setAccessible(true); kafkaSourceConfigField.set(source, kafkaSourceConfig); - Consumer consumer = mock(Consumer.class); + Consumer consumer = mock(Consumer.class); Mockito.doThrow(new RuntimeException("Pool exception")).when(consumer) .poll(Mockito.any(Duration.class)); From e7ae9e2544cced28eb4c2ae6c4b2f01a5879a318 Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Sun, 23 Jul 2023 08:36:41 +0800 Subject: [PATCH 12/29] [fix][broker] Fix potential OOM due to client use sasl authentication (#20713) Sasl authentication has two phases. In the first phase, the broker will temporarily cache the authentication info, and clean the cache at the end of the second phase. When the client crashes after phase 1, or someone deliberately attacks, the auth info will always remain in the broker's memory. If the client keeps reconnecting like this, the broker will soon OOM. --- .../AuthenticationProviderSasl.java | 14 +-- .../authentication/SaslAuthenticateTest.java | 85 ++++++++++++++++++- .../pulsar/broker/ServiceConfiguration.java | 12 +++ 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java index 93f7d3f420699..ed2d05ab7227f 100644 --- a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java +++ b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java @@ -33,6 +33,8 @@ import static org.apache.pulsar.common.sasl.SaslConstants.SASL_STATE_NEGOTIATE; import static org.apache.pulsar.common.sasl.SaslConstants.SASL_STATE_SERVER; import static org.apache.pulsar.common.sasl.SaslConstants.SASL_STATE_SERVER_CHECK_TOKEN; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import java.io.IOException; import java.net.SocketAddress; import java.net.URI; @@ -41,7 +43,7 @@ import java.util.Base64; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import javax.naming.AuthenticationException; @@ -72,6 +74,7 @@ public class AuthenticationProviderSasl implements AuthenticationProvider { private JAASCredentialsContainer jaasCredentialsContainer; private String loginContextName; + private Cache authStates; @Override public void initialize(ServiceConfiguration config) throws IOException { @@ -110,6 +113,9 @@ public void initialize(ServiceConfiguration config) throws IOException { throw new IllegalArgumentException(msg); } this.signer = new SaslRoleTokenSigner(secret); + this.authStates = Caffeine.newBuilder() + .maximumSize(config.getMaxInflightSaslContext()) + .expireAfterWrite(config.getInflightSaslContextExpiryMs(), TimeUnit.MILLISECONDS).build(); } @Override @@ -198,8 +204,6 @@ private byte[] readSecretFromUrl(String secretConfUrl) throws IOException { } } - private ConcurrentHashMap authStates = new ConcurrentHashMap<>(); - // return authState if it is in cache. private AuthenticationState getAuthState(HttpServletRequest request) { String id = request.getHeader(SASL_STATE_SERVER); @@ -208,7 +212,7 @@ private AuthenticationState getAuthState(HttpServletRequest request) { } try { - return authStates.get(Long.parseLong(id)); + return authStates.getIfPresent(Long.parseLong(id)); } catch (NumberFormatException e) { log.error("[{}] Wrong Id String in Token {}. e:", request.getRequestURI(), id, e); @@ -295,7 +299,7 @@ public boolean authenticateHttpRequest(HttpServletRequest request, HttpServletRe response.setStatus(HttpServletResponse.SC_OK); // auth completed, no need to keep authState - authStates.remove(state.getStateId()); + authStates.invalidate(state.getStateId()); return false; } else { // auth not complete diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java index 8a0d0392d1333..5cace2221dea8 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java @@ -18,26 +18,31 @@ */ package org.apache.pulsar.broker.authentication; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import com.github.benmanes.caffeine.cache.Cache; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.FileWriter; +import java.lang.reflect.Field; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Base64; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; - import javax.security.auth.login.Configuration; - -import com.google.common.collect.ImmutableSet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -59,6 +64,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import org.testng.collections.CollectionUtils; @Slf4j public class SaslAuthenticateTest extends ProducerConsumerBase { @@ -295,4 +301,75 @@ public void testSaslServerAndClientAuth() throws Exception { log.info("-- {} -- end", methodName); } + @Test + public void testSaslOnlyAuthFirstStage() throws Exception { + AuthenticationProviderSasl saslServer = (AuthenticationProviderSasl) pulsar.getBrokerService() + .getAuthenticationService().getAuthenticationProvider(SaslConstants.AUTH_METHOD_NAME); + + HttpServletRequest servletRequest = mock(HttpServletRequest.class); + doReturn("Init").when(servletRequest).getHeader("State"); + // 10 clients only do one-stage verification, resulting in 10 auth info remaining in memory + for (int i = 0; i < 10; i++) { + AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost"); + AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA); + doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when( + servletRequest).getHeader("SASL-Token"); + doReturn(String.valueOf(i)).when(servletRequest).getHeader("SASL-Server-ID"); + saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class)); + } + Field field = AuthenticationProviderSasl.class.getDeclaredField("authStates"); + field.setAccessible(true); + Cache cache = (Cache) field.get(saslServer); + assertEquals(cache.asMap().size(), 10); + // The cache expiration time is set to 1ms. Residual auth info should be cleaned up + conf.setInflightSaslContextExpiryMs(1); + saslServer.initialize(conf); + // Add more auth info into memory + for (int i = 0; i < 10; i++) { + AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost"); + AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA); + doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when( + servletRequest).getHeader("SASL-Token"); + doReturn(String.valueOf(10 + i)).when(servletRequest).getHeader("SASL-Server-ID"); + saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class)); + } + long start = System.currentTimeMillis(); + while (true) { + if (System.currentTimeMillis() - start > 10_00) { + fail(); + } + cache = (Cache) field.get(saslServer); + // Residual auth info should be cleaned up + if (CollectionUtils.hasElements(cache.asMap())) { + break; + } + Thread.yield(); + } + } + + @Test + public void testMaxInflightContext() throws Exception { + AuthenticationProviderSasl saslServer = (AuthenticationProviderSasl) pulsar.getBrokerService() + .getAuthenticationService().getAuthenticationProvider(SaslConstants.AUTH_METHOD_NAME); + HttpServletRequest servletRequest = mock(HttpServletRequest.class); + doReturn("Init").when(servletRequest).getHeader("State"); + conf.setInflightSaslContextExpiryMs(Integer.MAX_VALUE); + conf.setMaxInflightSaslContext(1); + saslServer.initialize(conf); + // add 10 inflight sasl context + for (int i = 0; i < 10; i++) { + AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost"); + AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA); + doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when( + servletRequest).getHeader("SASL-Token"); + doReturn(String.valueOf(i)).when(servletRequest).getHeader("SASL-Server-ID"); + saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class)); + } + Field field = AuthenticationProviderSasl.class.getDeclaredField("authStates"); + field.setAccessible(true); + Cache cache = (Cache) field.get(saslServer); + //only 1 context was left in the memory + assertEquals(cache.asMap().size(), 1); + } + } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 1a0ee2cfb756e..3d73734e82767 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1654,6 +1654,18 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private String kinitCommand = "/usr/bin/kinit"; + @FieldContext( + category = CATEGORY_SASL_AUTH, + doc = "how often the broker expires the inflight SASL context." + ) + private long inflightSaslContextExpiryMs = 30_000L; + + @FieldContext( + category = CATEGORY_SASL_AUTH, + doc = "Maximum number of inflight sasl context." + ) + private long maxInflightSaslContext = 50_000L; + /**** --- BookKeeper Client. --- ****/ @FieldContext( category = CATEGORY_STORAGE_BK, From 3c82d27193e1c687ec5f14b1a367ae6441d390dd Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sun, 23 Jul 2023 11:59:19 +0800 Subject: [PATCH 13/29] [fix][broker] Fix the built-in admin failed to delete a topic with a custom authz provider (#20848) --- .../pulsar/broker/authorization/AuthorizationProvider.java | 4 +--- .../pulsar/broker/auth/AuthorizationWithAuthDataTest.java | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java index 9ea49fee45c2c..24b94efb4882f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java @@ -436,9 +436,7 @@ default Boolean allowTopicPolicyOperation(TopicName topicName, * @return CompletableFuture */ default CompletableFuture removePermissionsAsync(TopicName topicName) { - return FutureUtil.failedFuture(new IllegalStateException( - String.format("removePermissionsAsync on topicName %s is not supported by the Authorization", - topicName))); + return CompletableFuture.completedFuture(null); } /** diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java index b69ccc40607f1..bb0461d321cb8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java @@ -270,6 +270,7 @@ public void testAdmin() throws PulsarAdminException { admin.topics().createNonPartitionedTopic(nonPartitionedTopic); admin.lookups().lookupPartitionedTopic(partitionedTopic); admin.lookups().lookupTopic(nonPartitionedTopic); + admin.topics().delete(nonPartitionedTopic); } @Test From 9256407cdca1ab6d9b3a59ce404dccb09953ab24 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Sun, 23 Jul 2023 12:56:04 +0800 Subject: [PATCH 14/29] [improve][broker] Avoid print redirect exception log when get list from bundle (#20846) --- .../broker/admin/v2/NonPersistentTopics.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index 36e88d91b3a83..c360eeabb5838 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -467,25 +467,20 @@ public void getListFromBundle( } asyncResponse.resume(topicList); }).exceptionally(ex -> { - Throwable realCause = FutureUtil.unwrapCompletionException(ex); - log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), - namespaceName, bundleRange, realCause); - if (realCause instanceof WebApplicationException) { - asyncResponse.resume(realCause); - } else { - asyncResponse.resume(new RestException(realCause)); + if (!isRedirectException(ex)) { + log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange, ex); } + resumeAsyncResponseExceptionally(asyncResponse, ex); return null; }); } }).exceptionally(ex -> { - log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), - namespaceName, bundleRange, ex); - if (ex.getCause() instanceof WebApplicationException) { - asyncResponse.resume(ex.getCause()); - } else { - asyncResponse.resume(new RestException(ex.getCause())); + if (!isRedirectException(ex)) { + log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange, ex); } + resumeAsyncResponseExceptionally(asyncResponse, ex); return null; }); } From 4ccb5bbee713267908f500f6a8d54ec1d66973d9 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sun, 23 Jul 2023 16:08:00 +0800 Subject: [PATCH 15/29] [improve][broker] Add the MessageExpirer interface to make code clear (#20800) Co-authored-by: tison --- .../admin/impl/PersistentTopicsBase.java | 66 ++++++------------- .../pulsar/broker/service/MessageExpirer.java | 30 +++++++++ .../pulsar/broker/service/Subscription.java | 6 +- .../PersistentMessageExpiryMonitor.java | 6 +- .../persistent/PersistentReplicator.java | 5 +- 5 files changed, 60 insertions(+), 53 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index a57722be4e1ce..682eb7b12e97e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -80,6 +80,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionInvalidCursorPosition; +import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; @@ -3955,28 +3956,20 @@ private CompletableFuture internalExpireMessagesByTimestampForSinglePartit } PersistentTopic topic = (PersistentTopic) t; - boolean issued; + final MessageExpirer messageExpirer; if (subName.startsWith(topic.getReplicatorPrefix())) { String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - PersistentReplicator repl = (PersistentReplicator) topic - .getPersistentReplicator(remoteCluster); - if (repl == null) { - resultFuture.completeExceptionally( - new RestException(Status.NOT_FOUND, "Replicator not found")); - return; - } - issued = repl.expireMessages(expireTimeInSeconds); + messageExpirer = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); } else { - PersistentSubscription sub = topic.getSubscription(subName); - if (sub == null) { - resultFuture.completeExceptionally( - new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return; - } - issued = sub.expireMessages(expireTimeInSeconds); + messageExpirer = topic.getSubscription(subName); + } + if (messageExpirer == null) { + final String message = subName.startsWith(topic.getReplicatorPrefix()) + ? "Replicator not found" : getSubNotFoundErrorMessage(topicName.toString(), subName); + resultFuture.completeExceptionally(new RestException(Status.NOT_FOUND, message)); + return; } - if (issued) { + if (messageExpirer.expireMessages(expireTimeInSeconds)) { log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), expireTimeInSeconds, topicName, subName); resultFuture.complete(null); @@ -4066,44 +4059,27 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit return; } try { - PersistentSubscription sub = null; - PersistentReplicator repl = null; - + final MessageExpirer messageExpirer; if (subName.startsWith(topic.getReplicatorPrefix())) { String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - repl = (PersistentReplicator) - topic.getPersistentReplicator(remoteCluster); - if (repl == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - "Replicator not found")); - return; - } + messageExpirer = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); } else { - sub = topic.getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return; - } + messageExpirer = topic.getSubscription(subName); + } + if (messageExpirer == null) { + final String message = (subName.startsWith(topic.getReplicatorPrefix())) + ? "Replicator not found" : getSubNotFoundErrorMessage(topicName.toString(), subName); + asyncResponse.resume(new RestException(Status.NOT_FOUND, message)); + return; } CompletableFuture batchSizeFuture = new CompletableFuture<>(); getEntryBatchSize(batchSizeFuture, topic, messageId, batchIndex); - PersistentReplicator finalRepl = repl; - PersistentSubscription finalSub = sub; - batchSizeFuture.thenAccept(bi -> { PositionImpl position = calculatePositionAckSet(isExcluded, bi, batchIndex, messageId); - boolean issued; try { - if (subName.startsWith(topic.getReplicatorPrefix())) { - issued = finalRepl.expireMessages(position); - } else { - issued = finalSub.expireMessages(position); - } - - if (issued) { + if (messageExpirer.expireMessages(position)) { log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), position, topicName, subName); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java new file mode 100644 index 0000000000000..7cb1d2a904aa4 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.common.classification.InterfaceStability; + +@InterfaceStability.Evolving +public interface MessageExpirer { + + boolean expireMessages(Position position); + + boolean expireMessages(int messageTTLInSeconds); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java index 1f418a6e9f85f..8ac855c1e7c37 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java @@ -30,7 +30,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; -public interface Subscription { +public interface Subscription extends MessageExpirer { BrokerInterceptor interceptor(); @@ -84,10 +84,6 @@ default long getNumberOfEntriesDelayed() { CompletableFuture peekNthMessage(int messagePosition); - boolean expireMessages(int messageTTLInSeconds); - - boolean expireMessages(Position position); - void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch); void redeliverUnacknowledgedMessages(Consumer consumer, List positions); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 3acd683f9f42b..020dc5323e55b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -32,16 +32,16 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.stats.Rate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** */ -public class PersistentMessageExpiryMonitor implements FindEntryCallback { +public class PersistentMessageExpiryMonitor implements FindEntryCallback, MessageExpirer { private final ManagedCursor cursor; private final String subName; private final PersistentTopic topic; @@ -73,6 +73,7 @@ public PersistentMessageExpiryMonitor(PersistentTopic topic, String subscription && this.cursor.getManagedLedger().getConfig().isAutoSkipNonRecoverableData(); } + @Override public boolean expireMessages(int messageTTLInSeconds) { if (expirationCheckInProgressUpdater.compareAndSet(this, FALSE, TRUE)) { log.info("[{}][{}] Starting message expiry check, ttl= {} seconds", topicName, subName, @@ -99,6 +100,7 @@ public boolean expireMessages(int messageTTLInSeconds) { } } + @Override public boolean expireMessages(Position messagePosition) { // If it's beyond last position of this topic, do nothing. PositionImpl topicLastPosition = (PositionImpl) this.topic.getLastPosition(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 92f4b3eaf9b2e..0d387da1dd912 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -47,6 +47,7 @@ import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; +import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; import org.apache.pulsar.client.api.MessageId; @@ -66,7 +67,7 @@ import org.slf4j.LoggerFactory; public abstract class PersistentReplicator extends AbstractReplicator - implements Replicator, ReadEntriesCallback, DeleteCallback { + implements Replicator, ReadEntriesCallback, DeleteCallback, MessageExpirer { protected final PersistentTopic topic; protected final ManagedCursor cursor; @@ -600,6 +601,7 @@ private long getReplicationDelayInSeconds() { return 0L; } + @Override public boolean expireMessages(int messageTTLInSeconds) { if ((cursor.getNumberOfEntriesInBacklog(false) == 0) || (cursor.getNumberOfEntriesInBacklog(false) < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK @@ -611,6 +613,7 @@ public boolean expireMessages(int messageTTLInSeconds) { return expiryMonitor.expireMessages(messageTTLInSeconds); } + @Override public boolean expireMessages(Position position) { return expiryMonitor.expireMessages(position); } From 69d7a2bf14555f11a716a9545c5cf391d8179a27 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 24 Jul 2023 10:43:07 +0800 Subject: [PATCH 16/29] [improve][broker] Add broker filter sync method back to guarantee the API compatibility (#20826) --- .../extensions/ExtensibleLoadManagerImpl.java | 2 +- .../filter/AntiAffinityGroupPolicyFilter.java | 2 +- .../extensions/filter/BrokerFilter.java | 22 ++++++++++++++--- .../filter/BrokerIsolationPoliciesFilter.java | 6 ++--- .../filter/BrokerLoadManagerClassFilter.java | 2 +- .../filter/BrokerMaxTopicCountFilter.java | 6 ++--- .../filter/BrokerVersionFilter.java | 6 ++--- .../extensions/scheduler/TransferShedder.java | 2 +- ...tiAffinityNamespaceGroupExtensionTest.java | 4 ++-- .../ExtensibleLoadManagerImplTest.java | 24 +++++++++---------- .../BrokerIsolationPoliciesFilterTest.java | 20 ++++++++-------- .../BrokerLoadManagerClassFilterTest.java | 4 ++-- .../filter/BrokerMaxTopicCountFilterTest.java | 2 +- .../filter/BrokerVersionFilterTest.java | 12 +++++----- .../scheduler/TransferShedderTest.java | 6 ++--- 15 files changed, 68 insertions(+), 52 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index f3c5e5dc2ee36..cba499eb8eedb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -493,7 +493,7 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, new ArrayList<>(filterPipeline.size()); for (final BrokerFilter filter : filterPipeline) { CompletableFuture> future = - filter.filter(availableBrokerCandidates, bundle, context); + filter.filterAsync(availableBrokerCandidates, bundle, context); futures.add(future); } CompletableFuture> result = new CompletableFuture<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java index df08ba96b9d7a..37b35d8661dd8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java @@ -39,7 +39,7 @@ public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) { } @Override - public CompletableFuture> filter( + public CompletableFuture> filterAsync( Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) { return helper.filterAsync(brokers, serviceUnitId.toString()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index 046cae48914f8..2950a0133899c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -42,8 +43,23 @@ public interface BrokerFilter { * @param context The load manager context. * @return Filtered broker list. */ - CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context); + @Deprecated + default Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + return filterAsync(brokers, serviceUnit, context).join(); + } + + /** + * Filter out async unqualified brokers based on implementation. + * + * @param brokers The full broker and lookup data. + * @param serviceUnit The current serviceUnit. + * @param context The load manager context. + * @return Filtered broker list. + */ + CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java index 0aa1dda437af7..306c4c36f4880 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -44,9 +44,9 @@ public String name() { } @Override - public CompletableFuture> filter(Map availableBrokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { return isolationPoliciesHelper.applyIsolationPoliciesAsync(availableBrokers, serviceUnit) .thenApply(brokerCandidateCache -> { availableBrokers.keySet().retainAll(brokerCandidateCache); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java index 6504da047a001..54d2a555e6103 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -34,7 +34,7 @@ public String name() { } @Override - public CompletableFuture> filter( + public CompletableFuture> filterAsync( Map brokers, ServiceUnitId serviceUnit, LoadManagerContext context) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index 044dcc83a7835..472cabf156636 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -36,9 +36,9 @@ public String name() { } @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 6e8ce5e58f2bb..1af39a6adb5fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -47,9 +47,9 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { return CompletableFuture.completedFuture(brokers); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index cec052b65ffaf..3564b4e9e3b94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -719,7 +719,7 @@ private boolean isTransferable(LoadManagerContext context, Map candidates = new HashMap<>(availableBrokers); for (var filter : brokerFilterPipeline) { try { - filter.filter(candidates, namespaceBundle, context) + filter.filterAsync(candidates, namespaceBundle, context) .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index b766c7de5a5f9..42fc12c2f998e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -130,7 +130,7 @@ public void testAntiAffinityGroupPolicyFilter() doReturn(namespace + "/" + bundle).when(namespaceBundle).toString(); var expected = new HashMap<>(brokers); - var actual = antiAffinityGroupPolicyFilter.filter( + var actual = antiAffinityGroupPolicyFilter.filterAsync( brokers, namespaceBundle, context).get(); assertEquals(actual, expected); @@ -141,7 +141,7 @@ public void testAntiAffinityGroupPolicyFilter() var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle.toString()) .get(5, TimeUnit.SECONDS).get(); expected.remove(srcBroker); - actual = antiAffinityGroupPolicyFilter.filter( + actual = antiAffinityGroupPolicyFilter.filterAsync( brokers, namespaceBundle, context).get(); assertEquals(actual, expected); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 4d15a5a5e2725..cbb3e174a0d87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -268,9 +268,9 @@ public String name() { } @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(pulsar1.getLookupServiceAddress()); return CompletableFuture.completedFuture(brokers); } @@ -289,9 +289,9 @@ public void testFilterHasException() throws Exception { doReturn(List.of(new MockBrokerFilter() { @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(brokers.keySet().iterator().next()); return FutureUtil.failedFuture(new BrokerFilterException("Test")); } @@ -532,17 +532,17 @@ public void testMoreThenOneFilter() throws Exception { String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); doReturn(List.of(new MockBrokerFilter() { @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(lookupServiceAddress1); return CompletableFuture.completedFuture(brokers); } },new MockBrokerFilter() { @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index b8d5d98fd4c08..f45e1405e1d21 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -83,20 +83,20 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper); // a. available-brokers: broker1, broker2, broker3 => result: broker1 - Map result = filter.filter(new HashMap<>(Map.of( + Map result = filter.filterAsync(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1")); // b. available-brokers: broker2, broker3 => result: broker2 - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker2", getLookupData(), "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker3", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); @@ -104,20 +104,20 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 2); // a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2 - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2")); // b. available-brokers: broker2, broker3 => result: broker2 - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker2", getLookupData(), "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker3", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); } @@ -141,14 +141,14 @@ public void testFilterWithPersistentOrNonPersistentDisabled() - Map result = filter.filter(new HashMap<>(Map.of( + Map result = filter.filterAsync(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker1", getLookupData(true, false), "broker2", getLookupData(true, false), "broker3", getLookupData())), namespaceBundle, getContext()).get(); @@ -156,13 +156,13 @@ public void testFilterWithPersistentOrNonPersistentDisabled() doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); - result = filter.filter(new HashMap<>(Map.of( + result = filter.filterAsync(new HashMap<>(Map.of( "broker1", getLookupData(false, true), "broker2", getLookupData(), "broker3", getLookupData())), namespaceBundle, getContext()).get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java index 6b35bf71a0317..17475b419576f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -49,14 +49,14 @@ public void test() throws BrokerFilterException, ExecutionException, Interrupted "broker5", getLookupData("3.0.0", null) ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); + Map result = filter.filterAsync(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()) )); context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); - result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); + result = filter.filterAsync(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index a6214304bd201..9d000cb91a155 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -60,7 +60,7 @@ public void test() throws IllegalAccessException, BrokerFilterException, Executi "broker4", getLookupData() ); Map result = - filter.filter(new HashMap<>(originalBrokers), null, context).get(); + filter.filterAsync(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index 4a1a45439384e..2aa7faeb599fd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -43,7 +43,7 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test public void testFilterEmptyBrokerList() throws BrokerFilterException, ExecutionException, InterruptedException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()).get(); + Map result = brokerVersionFilter.filterAsync(new HashMap<>(), null, getContext()).get(); assertTrue(result.isEmpty()); } @@ -60,7 +60,7 @@ public void testDisabledFilter() throws BrokerFilterException, ExecutionExceptio ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, null, context).get(); + Map result = brokerVersionFilter.filterAsync(brokers, null, context).get(); assertEquals(result, originalBrokers); } @@ -73,7 +73,7 @@ public void testFilter() throws BrokerFilterException, ExecutionException, Inter "localhost:6653", getLookupData("2.10.1") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter( + Map result = brokerVersionFilter.filterAsync( new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), @@ -87,7 +87,7 @@ public void testFilter() throws BrokerFilterException, ExecutionException, Inter "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + result = brokerVersionFilter.filterAsync(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -101,7 +101,7 @@ public void testFilter() throws BrokerFilterException, ExecutionException, Inter "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + result = brokerVersionFilter.filterAsync(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); @@ -115,7 +115,7 @@ public void testInvalidVersionString() { ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); try { - brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + brokerVersionFilter.filterAsync(new HashMap<>(originalBrokers), null, getContext()).get(); fail(); } catch (Exception ex) { assertEquals(ex.getCause().getClass(), BrokerFilterBadVersionException.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 3dd41a92e682c..26d95a0158d52 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -764,9 +764,9 @@ public String name() { } @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { return FutureUtil.failedFuture(new BrokerFilterException("test")); } }; From 563f929885ce24206d7bd164adadc7797f225106 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 25 Jul 2023 20:52:02 +0800 Subject: [PATCH 17/29] [fix][broker] Fix inconsensus namespace policies by `getPoliciesIfCached` (#20855) --- .../broker/resources/NamespaceResources.java | 7 ++ .../pulsar/broker/service/AbstractTopic.java | 20 +++- .../pulsar/broker/service/BrokerService.java | 50 +++++++- .../pulsar/broker/service/ServerCnx.java | 110 ++++++++++-------- .../nonpersistent/NonPersistentTopic.java | 6 +- .../persistent/DispatchRateLimiter.java | 6 + .../service/persistent/PersistentTopic.java | 12 +- .../persistent/SubscribeRateLimiter.java | 14 ++- 8 files changed, 155 insertions(+), 70 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index e5dd13c32eb23..f70d3f31d6e2f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -117,6 +117,13 @@ public Optional getPolicies(NamespaceName ns) throws MetadataStoreExce return get(joinPath(BASE_POLICIES_PATH, ns.toString())); } + /** + * Get the namespace policy from the metadata cache. This method will not trigger the load of metadata cache. + * + * @deprecated Since this method may introduce inconsistent namespace policies. we should use + * #{@link NamespaceResources#getPoliciesAsync} + */ + @Deprecated public Optional getPoliciesIfCached(NamespaceName ns) { return getCache().getIfCached(joinPath(BASE_POLICIES_PATH, ns.toString())); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 90693091c89b8..ec1593b076b44 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC; import com.google.common.base.MoreObjects; import java.util.ArrayList; @@ -40,6 +41,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.ToLongFunction; +import javax.annotation.Nonnull; import lombok.Getter; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.collections4.CollectionUtils; @@ -61,6 +63,7 @@ import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; @@ -1128,6 +1131,12 @@ public PublishRateLimiter getBrokerPublishRateLimiter() { return brokerService.getBrokerPublishRateLimiter(); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and we can use + * #{@link AbstractTopic#updateResourceGroupLimiter(Policies)} to instead of it. + */ + @Deprecated public void updateResourceGroupLimiter(Optional optPolicies) { Policies policies; try { @@ -1141,17 +1150,20 @@ public void updateResourceGroupLimiter(Optional optPolicies) { log.warn("[{}] Error getting policies {} and publish throttling will be disabled", topic, e.getMessage()); policies = new Policies(); } + updateResourceGroupLimiter(policies); + } + public void updateResourceGroupLimiter(@Nonnull Policies namespacePolicies) { + requireNonNull(namespacePolicies); // attach the resource-group level rate limiters, if set - String rgName = policies.resource_group_name; + String rgName = namespacePolicies.resource_group_name; if (rgName != null) { final ResourceGroup resourceGroup = - brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); + brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); if (resourceGroup != null) { this.resourceGroupRateLimitingEnabled = true; this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter(); - this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), - () -> this.enableCnxAutoRead()); + this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), this::enableCnxAutoRead); log.info("Using resource group {} rate limiter for topic {}", rgName, topic); return; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 8dc94ca874094..6c48e4d9ae89f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; import static org.apache.bookkeeper.mledger.ManagedLedgerConfig.PROPERTY_SOURCE_TOPIC_KEY; import static org.apache.commons.collections4.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -48,11 +49,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; @@ -68,6 +69,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import java.util.function.Predicate; +import javax.annotation.Nonnull; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; @@ -1934,7 +1936,7 @@ private void addTopicToStatsMaps(TopicName topicName, Topic topic) { } public void refreshTopicToStatsMaps(NamespaceBundle oldBundle) { - Objects.requireNonNull(oldBundle); + requireNonNull(oldBundle); try { // retrieve all topics under existing old bundle List topics = getAllTopicsFromNamespaceBundle(oldBundle.getNamespaceObject().toString(), @@ -3267,10 +3269,9 @@ public CompletableFuture isAllowAutoTopicCreationAsync(final String top } public CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName) { - Optional policies = - pulsar.getPulsarResources().getNamespaceResources() - .getPoliciesIfCached(topicName.getNamespaceObject()); - return isAllowAutoTopicCreationAsync(topicName, policies); + return pulsar.getPulsarResources().getNamespaceResources() + .getPoliciesAsync(topicName.getNamespaceObject()) + .thenCompose(policies -> isAllowAutoTopicCreationAsync(topicName, policies)); } private CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName, @@ -3340,11 +3341,23 @@ private AutoTopicCreationOverride getAutoTopicCreationOverride(final TopicName t return null; } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it. + */ + @Deprecated public boolean isAllowAutoSubscriptionCreation(final String topic) { TopicName topicName = TopicName.get(topic); return isAllowAutoSubscriptionCreation(topicName); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it. + */ + @Deprecated public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) { AutoSubscriptionCreationOverride autoSubscriptionCreationOverride = getAutoSubscriptionCreationOverride(topicName); @@ -3355,6 +3368,12 @@ public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) { } } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it. + */ + @Deprecated private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(final TopicName topicName) { Optional topicPolicies = getTopicPolicies(topicName); if (topicPolicies.isPresent() && topicPolicies.get().getAutoSubscriptionCreationOverride() != null) { @@ -3371,6 +3390,25 @@ private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(fin return null; } + public @Nonnull CompletionStage isAllowAutoSubscriptionCreationAsync(@Nonnull TopicName tpName) { + requireNonNull(tpName); + // topic level policies + final var topicPolicies = getTopicPolicies(tpName); + if (topicPolicies.isPresent() && topicPolicies.get().getAutoSubscriptionCreationOverride() != null) { + return CompletableFuture.completedFuture(topicPolicies.get().getAutoSubscriptionCreationOverride() + .isAllowAutoSubscriptionCreation()); + } + // namespace level policies + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(tpName.getNamespaceObject()) + .thenApply(policies -> { + if (policies.isPresent() && policies.get().autoSubscriptionCreationOverride != null) { + return policies.get().autoSubscriptionCreationOverride.isAllowAutoSubscriptionCreation(); + } + // broker level policies + return pulsar.getConfiguration().isAllowAutoSubscriptionCreation(); + }); + } + public boolean isSystemTopic(String topic) { return isSystemTopic(TopicName.get(topic)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 8dfeb54fd6a32..f3b355d63325f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -55,6 +55,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -1210,39 +1211,43 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .failedFuture(new TopicNotFoundException( "Topic " + topicName + " does not exist")); } + final Topic topic = optTopic.get(); + return service.isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowedAutoSubscriptionCreation -> { + boolean rejectSubscriptionIfDoesNotExist = isDurable + && !isAllowedAutoSubscriptionCreation + && !topic.getSubscriptions().containsKey(subscriptionName) + && topic.isPersistent(); + + if (rejectSubscriptionIfDoesNotExist) { + return FutureUtil + .failedFuture( + new SubscriptionNotFoundException( + "Subscription does not exist")); + } - Topic topic = optTopic.get(); - - boolean rejectSubscriptionIfDoesNotExist = isDurable - && !service.isAllowAutoSubscriptionCreation(topicName.toString()) - && !topic.getSubscriptions().containsKey(subscriptionName) - && topic.isPersistent(); - - if (rejectSubscriptionIfDoesNotExist) { - return FutureUtil - .failedFuture( - new SubscriptionNotFoundException( - "Subscription does not exist")); - } - - SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this) - .subscriptionName(subscriptionName) - .consumerId(consumerId).subType(subType).priorityLevel(priorityLevel) - .consumerName(consumerName).isDurable(isDurable) - .startMessageId(startMessageId).metadata(metadata).readCompacted(readCompacted) - .initialPosition(initialPosition) - .startMessageRollbackDurationSec(startMessageRollbackDurationSec) - .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) - .subscriptionProperties(subscriptionProperties) - .consumerEpoch(consumerEpoch) - .schemaType(schema == null ? null : schema.getType()) - .build(); - if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { - return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(v -> topic.subscribe(option)); - } else { - return topic.subscribe(option); - } + SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this) + .subscriptionName(subscriptionName) + .consumerId(consumerId).subType(subType) + .priorityLevel(priorityLevel) + .consumerName(consumerName).isDurable(isDurable) + .startMessageId(startMessageId).metadata(metadata) + .readCompacted(readCompacted) + .initialPosition(initialPosition) + .startMessageRollbackDurationSec(startMessageRollbackDurationSec) + .replicatedSubscriptionStateArg(isReplicated) + .keySharedMeta(keySharedMeta) + .subscriptionProperties(subscriptionProperties) + .consumerEpoch(consumerEpoch) + .schemaType(schema == null ? null : schema.getType()) + .build(); + if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { + return topic.addSchemaIfIdleOrCheckCompatible(schema) + .thenCompose(v -> topic.subscribe(option)); + } else { + return topic.subscribe(option); + } + }); }) .thenAccept(consumer -> { if (consumerFuture.complete(consumer)) { @@ -1461,33 +1466,38 @@ protected void handleProducer(final CommandProducer cmdProducer) { schemaVersionFuture.thenAccept(schemaVersion -> { topic.checkIfTransactionBufferRecoverCompletely(isTxnEnabled).thenAccept(future -> { - CompletableFuture createInitSubFuture; + CompletionStage createInitSubFuture; if (!Strings.isNullOrEmpty(initialSubscriptionName) && topic.isPersistent() && !topic.getSubscriptions().containsKey(initialSubscriptionName)) { - if (!this.getBrokerService().isAllowAutoSubscriptionCreation(topicName)) { - String msg = - "Could not create the initial subscription due to the auto subscription " - + "creation is not allowed."; - if (producerFuture.completeExceptionally( - new BrokerServiceException.NotAllowedException(msg))) { - log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", - remoteAddress, msg, initialSubscriptionName, topicName); - commandSender.sendErrorResponse(requestId, - ServerError.NotAllowedError, msg); - } - producers.remove(producerId, producerFuture); - return; - } - createInitSubFuture = - topic.createSubscription(initialSubscriptionName, InitialPosition.Earliest, - false, null); + createInitSubFuture = service.isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowAutoSubscriptionCreation -> { + if (!isAllowAutoSubscriptionCreation) { + return CompletableFuture.failedFuture( + new BrokerServiceException.NotAllowedException( + "Could not create the initial subscription due to" + + " the auto subscription creation is not allowed.")); + } + return topic.createSubscription(initialSubscriptionName, + InitialPosition.Earliest, false, null); + }); } else { createInitSubFuture = CompletableFuture.completedFuture(null); } createInitSubFuture.whenComplete((sub, ex) -> { if (ex != null) { + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + if (rc instanceof BrokerServiceException.NotAllowedException) { + log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", + remoteAddress, rc.getMessage(), initialSubscriptionName, topicName); + if (producerFuture.completeExceptionally(rc)) { + commandSender.sendErrorResponse(requestId, + ServerError.NotAllowedError, rc.getMessage()); + } + producers.remove(producerId, producerFuture); + return; + } String msg = "Failed to create the initial subscription: " + ex.getCause().getMessage(); log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 9fe0a735c90d9..c764283cb4459 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -167,17 +167,19 @@ public CompletableFuture initialize() { return brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) .thenCompose(optPolicies -> { + final Policies policies; if (!optPolicies.isPresent()) { log.warn("[{}] Policies not present and isEncryptionRequired will be set to false", topic); isEncryptionRequired = false; + policies = new Policies(); } else { - Policies policies = optPolicies.get(); + policies = optPolicies.get(); updateTopicPolicyByNamespacePolicy(policies); isEncryptionRequired = policies.encryption_required; isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; } updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(policies); return updateClusterMigrated(); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java index 0650d5bc1cf38..233972452e97f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java @@ -204,6 +204,12 @@ public static CompletableFuture> getPoliciesAsync(BrokerServi return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesAsync(namespace); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link DispatchRateLimiter#getPoliciesAsync(BrokerService, String)} to instead of it. + */ + @Deprecated public static Optional getPolicies(BrokerService brokerService, String topicName) { final NamespaceName namespace = TopicName.get(topicName).getNamespaceObject(); return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesIfCached(namespace); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index e907caa8c3000..039f576b04420 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled; import static org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic; @@ -340,7 +341,7 @@ public CompletableFuture initialize() { if (!optPolicies.isPresent()) { isEncryptionRequired = false; updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(new Policies()); initializeDispatchRateLimiterIfNeeded(); updateSubscribeRateLimiter(); return; @@ -356,7 +357,7 @@ public CompletableFuture initialize() { updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(policies); this.isEncryptionRequired = policies.encryption_required; @@ -487,7 +488,7 @@ public CompletableFuture unloadSubscription(@Nonnull String subName) { private PersistentSubscription createPersistentSubscription(String subscriptionName, ManagedCursor cursor, boolean replicated, Map subscriptionProperties) { - Objects.requireNonNull(topicCompactionService); + requireNonNull(topicCompactionService); if (isCompactionSubscription(subscriptionName) && topicCompactionService instanceof PulsarTopicCompactionService pulsarTopicCompactionService) { CompactedTopicImpl compactedTopic = pulsarTopicCompactionService.getCompactedTopic(); @@ -2804,7 +2805,8 @@ public void updateDispatchRateLimiter() { } @Override - public CompletableFuture onPoliciesUpdate(Policies data) { + public CompletableFuture onPoliciesUpdate(@Nonnull Policies data) { + requireNonNull(data); if (log.isDebugEnabled()) { log.debug("[{}] isEncryptionRequired changes: {} -> {}", topic, isEncryptionRequired, data.encryption_required); @@ -2826,7 +2828,7 @@ public CompletableFuture onPoliciesUpdate(Policies data) { updatePublishDispatcher(); - this.updateResourceGroupLimiter(Optional.of(data)); + updateResourceGroupLimiter(data); List> producerCheckFutures = new ArrayList<>(producers.size()); producers.values().forEach(producer -> producerCheckFutures.add( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java index a052f1f6abf33..58f099a9dab80 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java @@ -28,6 +28,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.util.RateLimiter; @@ -159,14 +160,21 @@ public void onSubscribeRateUpdate(SubscribeRate subscribeRate) { } /** - * Gets configured subscribe-rate from namespace policies. Returns null if subscribe-rate is not configured - * - * @return + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. */ + @Deprecated public SubscribeRate getPoliciesSubscribeRate() { return getPoliciesSubscribeRate(brokerService, topicName); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. + */ + @Deprecated public static SubscribeRate getPoliciesSubscribeRate(BrokerService brokerService, final String topicName) { final String cluster = brokerService.pulsar().getConfiguration().getClusterName(); final Optional policies = DispatchRateLimiter.getPolicies(brokerService, topicName); From 3f637680bbbc268d4801172d3212279ace38c4d4 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 25 Jul 2023 22:33:59 +0800 Subject: [PATCH 18/29] [fix][broker] Avoid infinite bundle unloading (#20822) --- .../impl/ModularLoadManagerImpl.java | 200 ++++++++++-------- .../ModularLoadManagerImplTest.java | 65 ++++-- 2 files changed, 166 insertions(+), 99 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/{ => impl}/ModularLoadManagerImplTest.java (92%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 73b4f318f3a36..8882c2cc6c56b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; @@ -659,11 +660,24 @@ public synchronized void doLoadShedding() { if (!shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) { return; } + NamespaceBundle bundleToUnload = LoadManagerShared.getNamespaceBundle(pulsar, bundle); + Optional destBroker = this.selectBroker(bundleToUnload); + if (!destBroker.isPresent()) { + log.info("[{}] No broker available to unload bundle {} from broker {}", + strategy.getClass().getSimpleName(), bundle, broker); + return; + } + if (destBroker.get().equals(broker)) { + log.warn("[{}] The destination broker {} is the same as the current owner broker for Bundle {}", + strategy.getClass().getSimpleName(), destBroker.get(), bundle); + return; + } - log.info("[{}] Unloading bundle: {} from broker {}", - strategy.getClass().getSimpleName(), bundle, broker); + log.info("[{}] Unloading bundle: {} from broker {} to dest broker {}", + strategy.getClass().getSimpleName(), bundle, broker, destBroker.get()); try { - pulsar.getAdminClient().namespaces().unloadNamespaceBundle(namespaceName, bundleRange); + pulsar.getAdminClient().namespaces() + .unloadNamespaceBundle(namespaceName, bundleRange, destBroker.get()); loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis()); } catch (PulsarServerException | PulsarAdminException e) { log.warn("Error when trying to perform load shedding on {} for broker {}", bundle, broker, e); @@ -837,99 +851,119 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni // If the given bundle is already in preallocated, return the selected broker. return Optional.of(preallocatedBundleToBroker.get(bundle)); } - final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); - brokerCandidateCache.clear(); - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - // filter brokers which owns topic higher than threshold - LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, - conf.getLoadBalancerBrokerMaxTopics()); - - // distribute namespaces to domain and brokers according to anti-affinity-group - LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, serviceUnit.toString(), - brokerCandidateCache, - brokerToNamespaceToBundleRange, brokerToFailureDomainMap); - - // distribute bundles evenly to candidate-brokers if enable - if (conf.isLoadBalancerDistributeBundlesEvenlyEnabled()) { - LoadManagerShared.removeMostServicingBrokersForNamespace(serviceUnit.toString(), - brokerCandidateCache, - brokerToNamespaceToBundleRange); - if (log.isDebugEnabled()) { - log.debug("enable distribute bundles evenly to candidate-brokers, broker candidate count={}", - brokerCandidateCache.size()); - } + Optional broker = selectBroker(serviceUnit); + if (!broker.isPresent()) { + // If no broker is selected, return empty. + return broker; } - log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle); + // Add new bundle to preallocated. + preallocateBundle(bundle, broker.get()); + return broker; + } + } finally { + selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + } + } - // Use the filter pipeline to finalize broker candidates. - try { - for (BrokerFilter filter : filterPipeline) { - filter.filter(brokerCandidateCache, data, loadData, conf); - } - } catch (BrokerFilterException x) { - // restore the list of brokers to the full set - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - } + private void preallocateBundle(String bundle, String broker) { + final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, + key -> getBundleDataOrDefault(bundle)); + loadData.getBrokerData().get(broker).getPreallocatedBundleData().put(bundle, data); + preallocatedBundleToBroker.put(bundle, broker); + + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + final ConcurrentOpenHashMap> namespaceToBundleRange = + brokerToNamespaceToBundleRange + .computeIfAbsent(broker, + k -> ConcurrentOpenHashMap.>newBuilder() + .build()); + synchronized (namespaceToBundleRange) { + namespaceToBundleRange.computeIfAbsent(namespaceName, + k -> ConcurrentOpenHashSet.newBuilder().build()) + .add(bundleRange); + } + } - if (brokerCandidateCache.isEmpty()) { - // restore the list of brokers to the full set - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - } + @VisibleForTesting + Optional selectBroker(final ServiceUnitId serviceUnit) { + synchronized (brokerCandidateCache) { + final String bundle = serviceUnit.toString(); + final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, + key -> getBundleDataOrDefault(bundle)); + brokerCandidateCache.clear(); + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + + // filter brokers which owns topic higher than threshold + LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, + conf.getLoadBalancerBrokerMaxTopics()); - // Choose a broker among the potentially smaller filtered list, when possible - Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + // distribute namespaces to domain and brokers according to anti-affinity-group + LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle, + brokerCandidateCache, + brokerToNamespaceToBundleRange, brokerToFailureDomainMap); + + // distribute bundles evenly to candidate-brokers if enable + if (conf.isLoadBalancerDistributeBundlesEvenlyEnabled()) { + LoadManagerShared.removeMostServicingBrokersForNamespace(bundle, + brokerCandidateCache, + brokerToNamespaceToBundleRange); if (log.isDebugEnabled()) { - log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); + log.debug("enable distribute bundles evenly to candidate-brokers, broker candidate count={}", + brokerCandidateCache.size()); } + } - if (!broker.isPresent()) { - // No brokers available - return broker; - } + log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle); - final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; - final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage(); - if (maxUsage > overloadThreshold) { - // All brokers that were in the filtered list were overloaded, so check if there is a better broker - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - Optional brokerTmp = - placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); - if (brokerTmp.isPresent()) { - broker = brokerTmp; - } + // Use the filter pipeline to finalize broker candidates. + try { + for (BrokerFilter filter : filterPipeline) { + filter.filter(brokerCandidateCache, data, loadData, conf); } + } catch (BrokerFilterException x) { + // restore the list of brokers to the full set + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + } - // Add new bundle to preallocated. - loadData.getBrokerData().get(broker.get()).getPreallocatedBundleData().put(bundle, data); - preallocatedBundleToBroker.put(bundle, broker.get()); - - final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); - final ConcurrentOpenHashMap> namespaceToBundleRange = - brokerToNamespaceToBundleRange - .computeIfAbsent(broker.get(), - k -> ConcurrentOpenHashMap.>newBuilder() - .build()); - synchronized (namespaceToBundleRange) { - namespaceToBundleRange.computeIfAbsent(namespaceName, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .add(bundleRange); - } + if (brokerCandidateCache.isEmpty()) { + // restore the list of brokers to the full set + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + } + + // Choose a broker among the potentially smaller filtered list, when possible + Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (log.isDebugEnabled()) { + log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); + } + + if (!broker.isPresent()) { + // No brokers available return broker; } - } finally { - selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + + final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; + final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage(); + if (maxUsage > overloadThreshold) { + // All brokers that were in the filtered list were overloaded, so check if there is a better broker + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + Optional brokerTmp = + placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (brokerTmp.isPresent()) { + broker = brokerTmp; + } + } + return broker; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java similarity index 92% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index 4b9f679f19d1f..3fb62f486ab8d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.loadbalance; +package org.apache.pulsar.broker.loadbalance.impl; import static java.lang.Thread.sleep; import static org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -55,11 +57,10 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.loadbalance.LoadBalancerTestingUtils; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; -import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Producer; @@ -410,39 +411,71 @@ public void testLoadShedding() throws Exception { doAnswer(invocation -> { bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]); return null; - }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + + AtomicReference> selectedBrokerRef = new AtomicReference<>(); + ModularLoadManagerImpl primaryLoadManagerSpy = spy(primaryLoadManager); + doAnswer(invocation -> { + ServiceUnitId serviceUnitId = (ServiceUnitId) invocation.getArguments()[0]; + Optional broker = primaryLoadManager.selectBroker(serviceUnitId); + selectedBrokerRef.set(broker); + return broker; + }).when(primaryLoadManagerSpy).selectBroker(any()); + setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1); pulsar1.getConfiguration().setLoadBalancerEnabled(true); - final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData"); + final LoadData loadData = (LoadData) getField(primaryLoadManagerSpy, "loadData"); final Map brokerDataMap = loadData.getBrokerData(); final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost)); when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData); brokerDataMap.put(primaryHost, brokerDataSpy1); // Need to update all the bundle data for the shredder to see the spy. - primaryLoadManager.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); + primaryLoadManagerSpy.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); sleep(100); localBrokerData.setCpu(new ResourceUsage(80, 100)); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // 80% is below overload threshold: verify nothing is unloaded. - verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(0)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); localBrokerData.setCpu(new ResourceUsage(90, 100)); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Most expensive bundle will be unloaded. - verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(1)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(2)); + assertEquals(selectedBrokerRef.get().get(), secondaryHost); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be // unloaded, but this is not the case due to the spy's behavior). - verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(2)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(1)); + assertEquals(selectedBrokerRef.get().get(), secondaryHost); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Now both are in grace period: neither should be unloaded. - verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(2)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + assertEquals(selectedBrokerRef.get().get(), secondaryHost); + + // Test bundle transfer to same broker + + loadData.getRecentlyUnloadedBundles().clear(); + primaryLoadManagerSpy.doLoadShedding(); + verify(namespacesSpy1, Mockito.times(3)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + + doReturn(Optional.of(primaryHost)).when(primaryLoadManagerSpy).selectBroker(any()); + loadData.getRecentlyUnloadedBundles().clear(); + primaryLoadManagerSpy.doLoadShedding(); + // The bundle shouldn't be unloaded because the broker is the same. + verify(namespacesSpy1, Mockito.times(3)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + } // Test that ModularLoadManagerImpl will determine that writing local data to ZooKeeper is necessary if certain From b12dbe782d82d859a68657f0464ba83bd04cb63e Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 26 Jul 2023 12:55:00 +0800 Subject: [PATCH 19/29] [improve][cli] Better error message for pulsar-admin schema upload (#20876) Signed-off-by: tison --- .../org/apache/pulsar/admin/cli/CmdSchemas.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java index 44ac143e3507e..638e9b1840a88 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java @@ -23,8 +23,10 @@ import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; +import java.io.FileNotFoundException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Path; import java.util.function.Supplier; import org.apache.pulsar.admin.cli.utils.SchemaExtractor; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -104,7 +106,19 @@ private class UploadSchema extends CliCommand { @Override void run() throws Exception { String topic = validateTopicName(params); - PostSchemaPayload input = MAPPER.readValue(new File(schemaFileName), PostSchemaPayload.class); + Path schemaPath = Path.of(schemaFileName); + File schemaFile = schemaPath.toFile(); + if (!schemaFile.exists()) { + final StringBuilder sb = new StringBuilder(); + sb.append("Schema file ").append(schemaPath).append(" is not found."); + if (!schemaPath.isAbsolute()) { + sb.append(" Relative path ").append(schemaPath) + .append(" is resolved to ").append(schemaPath.toAbsolutePath()) + .append(". Try to use absolute path if the relative one resolved wrongly."); + } + throw new FileNotFoundException(sb.toString()); + } + PostSchemaPayload input = MAPPER.readValue(schemaFile, PostSchemaPayload.class); getAdmin().schemas().createSchema(topic, input); } } From e55ed58eed4380b463d04b76dbb953d7a0ac795d Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 26 Jul 2023 13:16:27 +0800 Subject: [PATCH 20/29] [improve][broker] Add annotation for topic compaction strategy (#20858) ### Motivation Add `Audience.Private` annotation for topic compaction strategy. Since this interface only uses for load manager, other users should not use this interface. --- .../apache/pulsar/common/topics/TopicCompactionStrategy.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java index 51fb6742961bb..39bfa6d71bc96 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java @@ -21,6 +21,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; /** * Defines a custom strategy to compact messages in a topic. @@ -45,6 +47,8 @@ * "topicCompactionStrategyClassName", strategy.getClass().getCanonicalName())) * .create(); */ +@InterfaceAudience.Private +@InterfaceStability.Unstable public interface TopicCompactionStrategy { String TABLE_VIEW_TAG = "table-view"; From 362c4f4966a59bdb612ba4369d63bcfb23419e32 Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Wed, 26 Jul 2023 15:08:16 +0900 Subject: [PATCH 21/29] [fix][broker] Gracefully shutdown does not work with admin cli in standalone (#20709) Fixes : #20617 ### Motivation Currently, clients' shutdown API does not behave consistently in sense that asynchronicity is not handled explicitly. So issue #20617 happens. This PR will allow clients know that shutdown is actually triggered. ### Modifications - Synchronize call to `POST /shutdown` on client side - Asynchronize explicitly `pulsar().closeAsync()` invocation --- .../pulsar/broker/admin/impl/BrokersBase.java | 20 ++++++++--- .../apache/pulsar/client/admin/Brokers.java | 9 ++--- .../org/apache/pulsar/admin/cli/CmdBase.java | 34 +++++++++++++++++++ .../apache/pulsar/admin/cli/CmdBrokers.java | 4 +-- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index b367ce7aad955..be8390f15f826 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -545,16 +545,26 @@ public void shutDownBrokerGracefully( @ApiParam(name = "maxConcurrentUnloadPerSec", value = "if the value absent(value=0) means no concurrent limitation.") @QueryParam("maxConcurrentUnloadPerSec") int maxConcurrentUnloadPerSec, - @QueryParam("forcedTerminateTopic") @DefaultValue("true") boolean forcedTerminateTopic + @QueryParam("forcedTerminateTopic") @DefaultValue("true") boolean forcedTerminateTopic, + @Suspended final AsyncResponse asyncResponse ) { validateSuperUserAccess(); - doShutDownBrokerGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic); + doShutDownBrokerGracefullyAsync(maxConcurrentUnloadPerSec, forcedTerminateTopic) + .thenAccept(__ -> { + LOG.info("[{}] Successfully shutdown broker gracefully", clientAppId()); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + LOG.error("[{}] Failed to shutdown broker gracefully", clientAppId(), ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } - private void doShutDownBrokerGracefully(int maxConcurrentUnloadPerSec, - boolean forcedTerminateTopic) { + private CompletableFuture doShutDownBrokerGracefullyAsync(int maxConcurrentUnloadPerSec, + boolean forcedTerminateTopic) { pulsar().getBrokerService().unloadNamespaceBundlesGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic); - pulsar().closeAsync(); + return pulsar().closeAsync(); } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java index 464d02121cfc2..29c280f8ba536 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java @@ -326,10 +326,11 @@ Map getOwnedNamespaces(String cluster, String CompletableFuture healthcheckAsync(TopicVersion topicVersion); /** - * Shutdown current broker gracefully. - * @param maxConcurrentUnloadPerSec - * @param forcedTerminateTopic - * @return + * Trigger the current broker to graceful-shutdown asynchronously. + * + * @param maxConcurrentUnloadPerSec the maximum number of topics to unload per second. + * This helps control the speed of the unload operation during shutdown. + * @param forcedTerminateTopic if true, topics will be forcefully terminated during the shutdown process. */ CompletableFuture shutDownBrokerGracefully(int maxConcurrentUnloadPerSec, boolean forcedTerminateTopic); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java index ce2c44ec1e827..381bc8abcaa3f 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.admin.cli; +import static org.apache.pulsar.client.admin.internal.BaseResource.getApiException; import com.beust.jcommander.DefaultUsageFormatter; import com.beust.jcommander.IUsageFormatter; import com.beust.jcommander.JCommander; @@ -26,10 +27,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.PulsarAdminException.ConnectException; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; public abstract class CmdBase { protected final JCommander jcommander; @@ -37,6 +43,12 @@ public abstract class CmdBase { private PulsarAdmin admin; private IUsageFormatter usageFormatter; + /** + * Default read timeout in milliseconds. + * Used if not found from configuration data in {@link #getReadTimeoutMs()} + */ + private static final long DEFAULT_READ_TIMEOUT_MILLIS = 60000; + @Parameter(names = { "--help", "-h" }, help = true, hidden = true) private boolean help = false; @@ -124,6 +136,28 @@ protected PulsarAdmin getAdmin() { return admin; } + protected long getReadTimeoutMs() { + PulsarAdmin pulsarAdmin = getAdmin(); + if (pulsarAdmin instanceof PulsarAdminImpl) { + return ((PulsarAdminImpl) pulsarAdmin).getClientConfigData().getReadTimeoutMs(); + } + return DEFAULT_READ_TIMEOUT_MILLIS; + } + + protected T sync(Supplier> executor) throws PulsarAdminException { + try { + return executor.get().get(getReadTimeoutMs(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } catch (ExecutionException e) { + throw PulsarAdminException.wrap(getApiException(e.getCause())); + } catch (Exception e) { + throw PulsarAdminException.wrap(getApiException(e)); + } + } static Map parseListKeyValueMap(List metadata) { Map map = null; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java index 1e86edcf59c60..f1571e96c65c9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java @@ -150,8 +150,8 @@ private class ShutDownBrokerGracefully extends CliCommand { @Override void run() throws Exception { - getAdmin().brokers().shutDownBrokerGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic); - System.out.println("Successfully trigger broker shutdown gracefully"); + sync(() -> getAdmin().brokers().shutDownBrokerGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic)); + System.out.println("Successfully shutdown broker gracefully"); } } From ee47b92f8d56bb5b43ca105d9c1418839971e3bf Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 26 Jul 2023 16:02:00 +0800 Subject: [PATCH 22/29] [improve][ci] Add "pip" into the scope of the pull request title check rule (#20560) --- .github/workflows/ci-semantic-pull-request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-semantic-pull-request.yml b/.github/workflows/ci-semantic-pull-request.yml index 28246eb2b819c..9339b02771e5e 100644 --- a/.github/workflows/ci-semantic-pull-request.yml +++ b/.github/workflows/ci-semantic-pull-request.yml @@ -85,6 +85,7 @@ jobs: txn ws zk + pip # The pull request's title should be fulfilled the following pattern: # # [][] From ffb9d303f85cea0abd0839acb1cdcbddcc460b1d Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Wed, 26 Jul 2023 11:02:54 +0300 Subject: [PATCH 23/29] [fix][doc] Fix links to find older PIPs (#20882) --- pip/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pip/README.md b/pip/README.md index e88597e86e8a7..c268f8a57bc40 100644 --- a/pip/README.md +++ b/pip/README.md @@ -65,10 +65,10 @@ A PIP proposal can be in these states: The process works in the following way: 1. Fork https://github.com/apache/pulsar repository (Using the fork button on GitHub). -2. Clone the repository, and on it, copy the file `pip/TEMPLATE.md` and name it `pip-xxx.md`. The number `xxx` should be the next sequential number after the last contributed PIP. You view the list of contributed PIPs (at any status) as a list of Pull Requests having a title starting with `[pip][design] PIP-`. Use the link [here](https://github.com/apache/pulsar/pulls?q=is%3Apr+title%3A%22%5Bpip%5D%5Bdesign%5D+PIP-%22) as shortcut. +2. Clone the repository, and on it, copy the file `pip/TEMPLATE.md` and name it `pip-xxx.md`. The number `xxx` should be the next sequential number after the last contributed PIP. You view the list of contributed PIPs (at any status) as a list of Pull Requests having a "PIP" label. Use the link [here](https://github.com/apache/pulsar/pulls?q=is%3Apr+label%3APIP+) as shortcut. 3. Write the proposal following the section outlined by the template and the explanation for each section in the comment it contains (you can delete the comment once done). * If you need diagrams, avoid attaching large files. You can use [MermaidJS](https://mermaid.js.org/) as simple language to describe many types of diagrams. -4. Create GitHub Pull request (PR). The PR title should be `[pip][design] PIP-xxx: {title}`, where the `xxx` match the number given in previous step (file-name). Replace `{title}` with a short title to your proposal. +4. Create GitHub Pull request (PR). The PR title should be `[improve][pip] PIP-xxx: {title}`, where the `xxx` match the number given in previous step (file-name). Replace `{title}` with a short title to your proposal. 5. The author(s) will email the dev@pulsar.apache.org mailing list to kick off a discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion will happen in broader context either on the mailing list or as general comments on the PR. Many of the discussion items will be on particular aspect of the proposal, hence they should be as comments in the PR to specific lines in the proposal file. 6. Update file with a link to the discussion on the mailing. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org). 7. Based on the discussion and feedback, some changes might be applied by authors to the text of the proposal. They will be applied as extra commits, making it easier to track the changes. From a0372f4b91b4bd6600093cdb52e2db5d6ab8497e Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 27 Jul 2023 00:49:46 +0800 Subject: [PATCH 24/29] [fix][broker] Fix IndexOutOfBoundsException in the CompactedTopicUtils (#20887) --- .../compaction/CompactedTopicUtils.java | 1 + .../compaction/CompactedTopicUtilsTest.java | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java index cc5147c8e6623..6acd33279fd40 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java @@ -85,6 +85,7 @@ public static void asyncReadCompactedEntries(TopicCompactionService topicCompact } cursor.seek(seekToPosition); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); + return; } Entry lastEntry = entries.get(entries.size() - 1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java new file mode 100644 index 0000000000000..329abf9f780fa --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CompactedTopicUtilsTest { + + @Test + public void testReadCompactedEntriesWithEmptyEntries() throws ExecutionException, InterruptedException { + PositionImpl lastCompactedPosition = PositionImpl.get(1, 100); + TopicCompactionService service = Mockito.mock(TopicCompactionService.class); + Mockito.doReturn(CompletableFuture.completedFuture(Collections.emptyList())) + .when(service).readCompactedEntries(Mockito.any(), Mockito.intThat(argument -> argument > 0)); + Mockito.doReturn(CompletableFuture.completedFuture(lastCompactedPosition)).when(service) + .getLastCompactedPosition(); + + + PositionImpl initPosition = PositionImpl.get(1, 90); + AtomicReference readPositionRef = new AtomicReference<>(initPosition.getNext()); + ManagedCursor cursor = Mockito.mock(ManagedCursor.class); + Mockito.doReturn(readPositionRef.get()).when(cursor).getReadPosition(); + Mockito.doAnswer(invocation -> { + readPositionRef.set(invocation.getArgument(0)); + return null; + }).when(cursor).seek(Mockito.any()); + + CompletableFuture> completableFuture = new CompletableFuture<>(); + final AtomicReference throwableRef = new AtomicReference<>(); + AsyncCallbacks.ReadEntriesCallback readEntriesCallback = new AsyncCallbacks.ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + completableFuture.complete(entries); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + throwableRef.set(exception); + } + }; + + CompactedTopicUtils.asyncReadCompactedEntries(service, cursor, 1, 100, false, + readEntriesCallback, false, null); + + List entries = completableFuture.get(); + Assert.assertTrue(entries.isEmpty()); + Assert.assertNull(throwableRef.get()); + Assert.assertEquals(readPositionRef.get(), lastCompactedPosition.getNext()); + } +} From 5d72615b83578c357ca989369a90045308ff37d3 Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Fri, 28 Jul 2023 09:40:23 +0800 Subject: [PATCH 25/29] [pip][design] PIP-279: Reformat property in generateResponseWithEntry (#20627) --- pip/pip-279.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 pip/pip-279.md diff --git a/pip/pip-279.md b/pip/pip-279.md new file mode 100644 index 0000000000000..1a7f867118955 --- /dev/null +++ b/pip/pip-279.md @@ -0,0 +1,52 @@ +# Motivation + +reformat property,for a http header name cannot contain the following prohibited characters: =,;: \t\r\n\v\f + +for example: +{"city=shanghai":"tag"} +when we run `bin/pulsar-admin topics get-message-by-id `, it will throw exception, the exception is: +`Reason: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: a header name cannot contain the following prohibited characters: =,;: \t\r\n\v\f: =` + +# High Level Design + +In master branch, +in an http request:getMessageById("/{tenant}/{namespace}/{topic}/ledger/{ledgerId}/entry/{entryId}"), +replace `"X-Pulsar-PROPERTY-" + msgProperties.getKey()` with `"X-Pulsar-PROPERTY"` + +After release-3.1.0, this feature begins to take effect. + +# Concrete Example + +for example, the current message's properties likes this: +``` +"name": "James" +"gender": "man" +"details=man": "good at playing basketball" +``` + +## BEFORE +old response header format: +``` +headers: { + "X-Pulsar-PROPERTY-name": "James", + "X-Pulsar-PROPERTY-gender": "man", + "X-Pulsar-PROPERTY-details=man": "good at playing basketball" +} +``` +but it will throw exception in the end check + +## AFTER +new response header format: +``` +headers: { +"X-Pulsar-PROPERTY": '{"name": "James", "gender": "man", "details=man": "good at playing basketball"}' +} +``` + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/vfc99mbj2z2xgwfs1hq1zxrow13qm2n7 +* Mailing List voting thread: https://lists.apache.org/thread/g354684m9h495o3p0kmzb7fh7vfxhddx From 9be0b52ac425602cdb0bdd9348d0b63c16f6f62a Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 28 Jul 2023 10:43:34 +0800 Subject: [PATCH 26/29] [fix][broker] In replication scenario, remote consumer could not be registered if there has no message was sent (#20888) Motivation: In the replication scenario, we want to produce messages on the native cluster and consume messages on the remote cluster, the producer and consumer both use a same schema, but the consumer cannot be registered if there has no messages in the topic yet.The root cause is that for the remote cluster, there is a producer who has been registered with `AUTO_PRODUCE_BYTES` schema, so there is no schema to check the compatibility. Modifications: If there is no schema and only the replicator producer was registered, skip the compatibility check. --- .../pulsar/broker/service/AbstractTopic.java | 2 +- .../service/persistent/PersistentTopic.java | 2 +- .../broker/service/OneWayReplicatorTest.java | 30 +++++++++++++++++++ .../org/apache/pulsar/schema/SchemaTest.java | 27 +++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index ec1593b076b44..b15f8cbf0b848 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -139,7 +139,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener USER_CREATED_PRODUCER_COUNTER_UPDATER = AtomicIntegerFieldUpdater.newUpdater(AbstractTopic.class, "userCreatedProducerCount"); - private volatile int userCreatedProducerCount = 0; + protected volatile int userCreatedProducerCount = 0; protected volatile Optional topicEpoch = Optional.empty(); private volatile boolean hasExclusiveProducer; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 039f576b04420..2b1700f36cb68 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3272,7 +3272,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .toList().size()) .sum(); if (hasSchema - || (!producers.isEmpty()) + || (userCreatedProducerCount > 0) || (numActiveConsumersWithoutAutoSchema != 0) || (ledger.getTotalSize() != 0)) { return checkSchemaCompatibleForConsumer(schema); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index b1bc2b993f96e..e068cd55e9055 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -19,13 +19,18 @@ package org.apache.pulsar.broker.service; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.util.Optional; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.policies.data.TopicStats; +import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -75,4 +80,29 @@ public void testReplicatorProducerStatInTopic() throws Exception { admin2.topics().delete(topicName); }); } + + @Test + public void testCreateRemoteConsumerFirst() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); + // Wait for replicator started. + Awaitility.await().untilAsserted(() -> { + Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional2.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + }); + // The topic in cluster2 has a replicator created producer(schema Auto_Produce), but does not have any schema。 + // Verify: the consumer of this cluster2 can create successfully. + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topicName).subscriptionName("s1") + .subscribe();; + + // cleanup. + producer1.close(); + consumer2.close(); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 7ba4529cdbdf4..7eae6462545c8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -1290,6 +1290,33 @@ private void testIncompatibleSchema() throws Exception { assertThrows(SchemaSerializationException.class, message2::getValue); } + /** + * This test just ensure that schema check still keeps the original logic: if there has any producer, but no schema + * was registered, the new consumer could not register new schema. + * TODO: I think this design should be improved: if a producer used "AUTO_PRODUCE_BYTES" schema, we should allow + * the new consumer to register new schema. But before we can solve this problem, we need to modify + * "CmdProducer" to let the Broker know that the Producer uses a schema of type "AUTO_PRODUCE_BYTES". + */ + @Test + public void testAutoProduceAndSpecifiedConsumer() throws Exception { + final String namespace = PUBLIC_TENANT + "/ns_" + randomName(16); + admin.namespaces().createNamespace(namespace, Sets.newHashSet(CLUSTER_NAME)); + final String topicName = "persistent://" + namespace + "/tp_" + randomName(16); + admin.topics().createNonPartitionedTopic(topicName); + + Producer producer = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName).create(); + try { + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub1").subscribe(); + fail("Should throw ex: Topic does not have schema to check"); + } catch (Exception ex){ + assertTrue(ex.getMessage().contains("Topic does not have schema to check")); + } + + // Cleanup. + producer.close(); + admin.topics().delete(topicName); + } + @Test public void testCreateSchemaInParallel() throws Exception { final String namespace = "test-namespace-" + randomName(16); From ca01447fb4df808f8e2da2dc75d44fad0b780032 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:10:35 +0800 Subject: [PATCH 27/29] [improve][txn]Add log when can't send TcClientConnectRequest to broker. (#20880) Main Issue: https://github.com/apache/pulsar/issues/20851 ### Motivation When the Proto version does not allow us to send TcClientConnectRequest to the broker, we should add a log to debug it. ### Modifications Add a waining log. --- .../apache/pulsar/client/impl/TransactionMetaStoreHandler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index ebbfca0c3cb3f..0b5174a015118 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -167,6 +167,8 @@ public CompletableFuture connectionOpened(ClientCnx cnx) { return null; }); } else { + LOG.warn("Can not connect to the transaction coordinator because the protocol version {} is " + + "lower than 19", cnx.getRemoteEndpointProtocolVersion()); registerToConnection(cnx); future.complete(null); } From 001b134653b89103089b1429b9058d987cd0485c Mon Sep 17 00:00:00 2001 From: crossoverJie Date: Mon, 31 Jul 2023 20:40:29 +0800 Subject: [PATCH 28/29] [improve][sql] Fix the wrong format of the logs (#20907) --- .../java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java | 2 +- .../apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 858624b156dbb..6663b349761a3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -776,7 +776,7 @@ private void initEntryCacheSizeAllocator(PulsarConnectorConfig connectorConfig) connectorConfig.getMaxSplitQueueSizeBytes() / 2); this.messageQueueCacheSizeAllocator = new NoStrictCacheSizeAllocator( connectorConfig.getMaxSplitQueueSizeBytes() / 2); - log.info("Init cacheSizeAllocator with maxSplitEntryQueueSizeBytes {}.", + log.info("Init cacheSizeAllocator with maxSplitEntryQueueSizeBytes %d.", connectorConfig.getMaxSplitQueueSizeBytes()); } else { this.entryQueueCacheSizeAllocator = new NullCacheSizeAllocator(); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java index e2d030d2d7f1b..673ea2b3940d9 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java @@ -74,7 +74,7 @@ public CompletableFuture getSchemaByVersion(byte[] schemaVersion) { } return cache.get(BytesSchemaVersion.of(schemaVersion)); } catch (ExecutionException e) { - LOG.error("Can't get generic schema for topic {} schema version {}", + LOG.error("Can't get generic schema for topic %s schema version %s", topicName.toString(), new String(schemaVersion, StandardCharsets.UTF_8), e); return FutureUtil.failedFuture(e.getCause()); } From b8d3adeeeb6c127a361ab734865f344d00d306b6 Mon Sep 17 00:00:00 2001 From: StreamNative Bot Date: Mon, 31 Jul 2023 21:06:45 +0000 Subject: [PATCH 29/29] Release 3.1.0-SNAPSHOT --- .idea/vcs.xml | 28 +- .mvn/extensions.xml | 5 +- .mvn/gradle-enterprise.xml | 6 +- bouncy-castle/bc/pom.xml | 14 +- bouncy-castle/bcfips-include-test/pom.xml | 12 +- bouncy-castle/bcfips/pom.xml | 12 +- bouncy-castle/pom.xml | 10 +- buildtools/pom.xml | 75 +- buildtools/src/main/resources/log4j2.xml | 32 +- .../src/main/resources/pulsar/checkstyle.xml | 580 +++-- .../main/resources/pulsar/suppressions.xml | 121 +- conf/filesystem_offload_core_site.xml | 52 +- conf/functions_log4j2.xml | 217 +- distribution/io/pom.xml | 36 +- distribution/io/src/assemble/io.xml | 155 +- distribution/offloaders/pom.xml | 34 +- .../offloaders/src/assemble/offloaders.xml | 7 +- distribution/pom.xml | 36 +- distribution/server/pom.xml | 64 +- distribution/server/src/assemble/bin.xml | 12 +- distribution/shell/pom.xml | 38 +- distribution/shell/src/assemble/shell.xml | 7 +- docker/pom.xml | 31 +- docker/pulsar-all/pom.xml | 10 +- docker/pulsar/pom.xml | 11 +- jclouds-shaded/pom.xml | 31 +- managed-ledger/pom.xml | 34 +- pom.xml | 542 ++--- pulsar-broker-auth-athenz/pom.xml | 19 +- .../src/test/resources/findbugsExclude.xml | 3 +- pulsar-broker-auth-oidc/pom.xml | 24 +- pulsar-broker-auth-sasl/pom.xml | 21 +- pulsar-broker-common/pom.xml | 21 +- pulsar-broker/pom.xml | 96 +- pulsar-client-1x-base/pom.xml | 12 +- .../pulsar-client-1x/pom.xml | 15 +- .../src/main/resources/findbugsExclude.xml | 1 + .../pulsar-client-2x-shaded/pom.xml | 43 +- pulsar-client-admin-api/pom.xml | 134 +- pulsar-client-admin-shaded/pom.xml | 131 +- pulsar-client-admin/pom.xml | 24 +- pulsar-client-all/pom.xml | 77 +- pulsar-client-api/pom.xml | 145 +- .../src/main/resources/findbugsExclude.xml | 5 +- pulsar-client-auth-athenz/pom.xml | 20 +- .../src/test/resources/findbugsExclude.xml | 1 + pulsar-client-auth-sasl/pom.xml | 20 +- pulsar-client-messagecrypto-bc/pom.xml | 12 +- pulsar-client-shaded/pom.xml | 59 +- pulsar-client-tools-api/pom.xml | 10 +- .../pom.xml | 5 +- pulsar-client-tools-test/pom.xml | 13 +- .../src/test/resources/findbugsExclude.xml | 1 + pulsar-client-tools/pom.xml | 17 +- pulsar-client/pom.xml | 37 +- .../src/main/resources/findbugsExclude.xml | 1968 ++++++++--------- pulsar-common/pom.xml | 50 +- .../src/main/resources/findbugsExclude.xml | 66 +- pulsar-config-validation/pom.xml | 120 +- pulsar-docs-tools/pom.xml | 54 +- pulsar-functions/api-java/pom.xml | 28 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-functions/instance/pom.xml | 40 +- .../src/main/resources/findbugsExclude.xml | 1 + .../java-examples-builtin/pom.xml | 34 +- pulsar-functions/java-examples/pom.xml | 11 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-functions/localrun-shaded/pom.xml | 825 +++---- pulsar-functions/localrun/pom.xml | 157 +- .../src/main/resources/findbugsExclude.xml | 1 + .../localrun/src/main/resources/log4j2.xml | 24 +- pulsar-functions/pom.xml | 45 +- pulsar-functions/proto/pom.xml | 138 +- pulsar-functions/runtime-all/pom.xml | 23 +- .../src/main/resources/findbugsExclude.xml | 21 +- .../main/resources/java_instance_log4j2.xml | 219 +- .../resources/kubernetes_instance_log4j2.xml | 79 +- pulsar-functions/runtime/pom.xml | 19 +- .../src/main/resources/findbugsExclude.xml | 21 +- pulsar-functions/secrets/pom.xml | 11 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-functions/utils/pom.xml | 30 +- pulsar-functions/worker/pom.xml | 40 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-io/aerospike/pom.xml | 16 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-io/alluxio/pom.xml | 178 +- pulsar-io/aws/pom.xml | 15 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-io/batch-data-generator/pom.xml | 146 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/batch-discovery-triggerers/pom.xml | 14 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/canal/pom.xml | 206 +- pulsar-io/cassandra/pom.xml | 14 +- pulsar-io/common/pom.xml | 93 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/core/pom.xml | 10 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/data-generator/pom.xml | 88 +- pulsar-io/debezium/core/pom.xml | 21 +- pulsar-io/debezium/mongodb/pom.xml | 80 +- pulsar-io/debezium/mssql/pom.xml | 14 +- pulsar-io/debezium/mysql/pom.xml | 15 +- pulsar-io/debezium/oracle/pom.xml | 14 +- pulsar-io/debezium/pom.xml | 10 +- pulsar-io/debezium/postgres/pom.xml | 15 +- pulsar-io/docs/pom.xml | 12 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/dynamodb/pom.xml | 21 +- pulsar-io/elastic-search/pom.xml | 19 +- pulsar-io/file/pom.xml | 26 +- pulsar-io/flume/pom.xml | 266 ++- pulsar-io/hbase/pom.xml | 206 +- .../src/test/resources/hbase/hbase-site.xml | 41 +- pulsar-io/hdfs2/pom.xml | 114 +- .../src/main/resources/findbugsExclude.xml | 1 + .../src/test/resources/hadoop/core-site.xml | 17 +- .../src/test/resources/hadoop/hdfs-site.xml | 25 +- pulsar-io/hdfs3/pom.xml | 54 +- .../src/main/resources/findbugsExclude.xml | 1 + .../src/test/resources/hadoop/core-site.xml | 17 +- .../src/test/resources/hadoop/hdfs-site.xml | 25 +- pulsar-io/http/pom.xml | 119 +- pulsar-io/influxdb/pom.xml | 128 +- pulsar-io/jdbc/clickhouse/pom.xml | 14 +- pulsar-io/jdbc/core/pom.xml | 18 +- pulsar-io/jdbc/mariadb/pom.xml | 11 +- pulsar-io/jdbc/openmldb/pom.xml | 10 +- pulsar-io/jdbc/pom.xml | 8 +- pulsar-io/jdbc/postgres/pom.xml | 10 +- pulsar-io/jdbc/sqlite/pom.xml | 11 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 35 +- pulsar-io/kafka-connect-adaptor/pom.xml | 34 +- pulsar-io/kafka/pom.xml | 23 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/kinesis/pom.xml | 33 +- pulsar-io/mongo/pom.xml | 16 +- pulsar-io/netty/pom.xml | 144 +- .../src/main/resources/findbugsExclude.xml | 25 +- pulsar-io/nsq/pom.xml | 27 +- pulsar-io/pom.xml | 14 +- pulsar-io/rabbitmq/pom.xml | 17 +- pulsar-io/redis/pom.xml | 139 +- pulsar-io/solr/pom.xml | 154 +- .../src/main/resources/findbugsExclude.xml | 45 +- pulsar-io/solr/src/test/resources/solr.xml | 34 +- pulsar-io/twitter/pom.xml | 24 +- pulsar-metadata/pom.xml | 23 +- .../src/main/resources/findbugsExclude.xml | 25 +- .../src/test/resources/findbugsExclude.xml | 19 +- .../bookkeeper-storage/pom.xml | 151 +- pulsar-package-management/core/pom.xml | 152 +- .../src/main/resources/findbugsExclude.xml | 1 + .../filesystem-storage/pom.xml | 61 +- pulsar-package-management/pom.xml | 74 +- pulsar-proxy/pom.xml | 38 +- pulsar-sql/pom.xml | 290 ++- pulsar-sql/presto-distribution/pom.xml | 33 +- .../src/assembly/assembly.xml | 121 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 131 +- .../src/assembly/assembly.xml | 37 +- pulsar-sql/presto-pulsar/pom.xml | 442 ++-- pulsar-testclient/pom.xml | 303 ++- pulsar-transaction/common/pom.xml | 83 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-transaction/coordinator/pom.xml | 206 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-transaction/pom.xml | 13 +- pulsar-websocket/pom.xml | 31 +- .../src/main/resources/findbugsExclude.xml | 4 +- src/assembly-source-package.xml | 10 +- src/check-binary-license.sh | 4 +- src/findbugs-exclude.xml | 28 +- src/idea-code-style.xml | 71 +- ...owasp-dependency-check-false-positives.xml | 76 +- src/owasp-dependency-check-suppressions.xml | 747 ++++--- src/settings.xml | 3 +- structured-event-log/pom.xml | 12 +- testmocks/pom.xml | 17 +- tests/bc_2_0_0/pom.xml | 160 +- tests/bc_2_0_0/src/test/resources/pulsar.xml | 15 +- tests/bc_2_0_1/pom.xml | 160 +- tests/bc_2_0_1/src/test/resources/pulsar.xml | 15 +- tests/bc_2_6_0/pom.xml | 174 +- .../src/test/resources/backwards-client.xml | 15 +- .../docker-images/java-test-functions/pom.xml | 18 +- tests/docker-images/java-test-image/pom.xml | 15 +- tests/docker-images/java-test-plugins/pom.xml | 9 +- .../latest-version-image/pom.xml | 17 +- tests/docker-images/pom.xml | 8 +- tests/integration/pom.xml | 48 +- .../src/test/resources/pulsar-auth.xml | 9 +- .../pulsar-backwards-compatibility.xml | 29 +- .../src/test/resources/pulsar-cli.xml | 35 +- .../src/test/resources/pulsar-function.xml | 19 +- .../test/resources/pulsar-io-ora-source.xml | 13 +- .../src/test/resources/pulsar-io-sinks.xml | 15 +- .../src/test/resources/pulsar-io-sources.xml | 19 +- .../src/test/resources/pulsar-loadbalance.xml | 15 +- .../src/test/resources/pulsar-messaging.xml | 27 +- .../src/test/resources/pulsar-plugin.xml | 19 +- .../src/test/resources/pulsar-process.xml | 20 +- .../src/test/resources/pulsar-proxy.xml | 15 +- .../src/test/resources/pulsar-python.xml | 13 +- .../src/test/resources/pulsar-schema.xml | 17 +- .../src/test/resources/pulsar-semantics.xml | 13 +- .../src/test/resources/pulsar-sql.xml | 17 +- .../src/test/resources/pulsar-standalone.xml | 15 +- .../src/test/resources/pulsar-thread.xml | 22 +- .../src/test/resources/pulsar-tls.xml | 13 +- .../src/test/resources/pulsar-transaction.xml | 13 +- .../src/test/resources/pulsar-upgrade.xml | 13 +- .../src/test/resources/pulsar-websocket.xml | 15 +- .../integration/src/test/resources/pulsar.xml | 41 +- .../resources/tiered-filesystem-storage.xml | 15 +- .../test/resources/tiered-jcloud-storage.xml | 15 +- tests/pom.xml | 11 +- tests/pulsar-client-admin-shade-test/pom.xml | 187 +- .../src/test/resources/pulsar.xml | 15 +- tests/pulsar-client-all-shade-test/pom.xml | 185 +- .../src/test/resources/pulsar.xml | 15 +- tests/pulsar-client-shade-test/pom.xml | 175 +- .../src/test/resources/pulsar.xml | 15 +- tiered-storage/file-system/pom.xml | 383 ++-- .../src/main/resources/findbugsExclude.xml | 1 + tiered-storage/jcloud/pom.xml | 17 +- .../src/main/resources/findbugsExclude.xml | 1 + tiered-storage/pom.xml | 9 +- 229 files changed, 7163 insertions(+), 8284 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 99fce8b2b9812..951f00c3cc3b8 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,4 +1,4 @@ - + - - - - - - + + + + + + diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 872764f899827..ca4639f8053e0 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -1,4 +1,4 @@ - + - + com.gradle gradle-enterprise-maven-extension diff --git a/.mvn/gradle-enterprise.xml b/.mvn/gradle-enterprise.xml index 2667402c23cdb..c8626080d0866 100644 --- a/.mvn/gradle-enterprise.xml +++ b/.mvn/gradle-enterprise.xml @@ -1,4 +1,4 @@ - + - + https://ge.apache.org false diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 7ea333a4711a7..632a83b9fe731 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent 3.1.0-SNAPSHOT .. - bouncy-castle-bc Apache Pulsar :: Bouncy Castle :: BC - ${project.groupId} @@ -39,23 +36,19 @@ ${project.version} provided - org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} - org.bouncycastle bcprov-ext-jdk18on ${bouncycastle.version} - - de.ntcomputer @@ -73,7 +66,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index e8348be9292cd..7c102ee9e23e3 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative bouncy-castle-parent 3.1.0-SNAPSHOT .. - bcfips-include-test Pulsar Bouncy Castle FIPS Test Broker and client runs auth include BC FIPS verison - ${project.groupId} @@ -39,7 +37,6 @@ ${project.version} test - ${project.groupId} pulsar-broker @@ -53,7 +50,6 @@ test-jar test - ${project.groupId} pulsar-broker @@ -66,7 +62,6 @@ test - ${project.groupId} @@ -74,7 +69,6 @@ ${project.version} pkg - diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index a07e5e19907f2..645074feab382 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent 3.1.0-SNAPSHOT .. - bouncy-castle-bcfips Apache Pulsar :: Bouncy Castle :: BC-FIPS - ${project.groupId} @@ -39,20 +36,17 @@ ${project.version} provided - org.bouncycastle bc-fips ${bouncycastle.bc-fips.version} - org.bouncycastle bcpkix-fips ${bouncycastle.bcpkix-fips.version} - diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index daefeb83b5371..f69048ede3265 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - @@ -42,16 +41,13 @@ - bouncy-castle-parent Apache Pulsar :: Bouncy Castle :: Parent - bc bcfips - bcfips-include-test diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 836e1c5cb5f0e..01ceaac638812 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache apache 29 - + - - org.apache.pulsar + io.streamnative buildtools 3.1.0-SNAPSHOT jar Pulsar Build Tools - 2023-05-03T02:53:27Z 1.8 @@ -60,7 +57,6 @@ --add-opens java.base/jdk.internal.platform=ALL-UNNAMED - @@ -72,9 +68,7 @@ - - org.yaml snakeyaml @@ -95,7 +89,6 @@ guice ${guice.version} - org.testng testng @@ -147,7 +140,6 @@ ${mockito.version} - @@ -256,6 +248,55 @@ **/proto/* + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + @@ -265,4 +306,14 @@ + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml index 184f58487eaf0..9e12d754fe7b5 100644 --- a/buildtools/src/main/resources/log4j2.xml +++ b/buildtools/src/main/resources/log4j2.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml index c63c8993408de..7a32584cc4dd6 100644 --- a/buildtools/src/main/resources/pulsar/checkstyle.xml +++ b/buildtools/src/main/resources/pulsar/checkstyle.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + - - - - + IMPORT CHECKS - - - - + --> + + + + - - - - - + + + + + + + - - - - + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + } else --> - else --> - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - + + + diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml index 57a01c60f6a27..e558a8a0618fd 100644 --- a/buildtools/src/main/resources/pulsar/suppressions.xml +++ b/buildtools/src/main/resources/pulsar/suppressions.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/filesystem_offload_core_site.xml b/conf/filesystem_offload_core_site.xml index d26cec2cc60f0..3c7e6d94bbe50 100644 --- a/conf/filesystem_offload_core_site.xml +++ b/conf/filesystem_offload_core_site.xml @@ -1,3 +1,4 @@ + - - - fs.defaultFS - - - - hadoop.tmp.dir - pulsar - - - io.file.buffer.size - 4096 - - - io.seqfile.compress.blocksize - 1000000 - - - io.seqfile.compression.type - BLOCK - - - io.map.index.interval - 128 - - + + + fs.defaultFS + + + + hadoop.tmp.dir + pulsar + + + io.file.buffer.size + 4096 + + + io.seqfile.compress.blocksize + 1000000 + + + io.seqfile.compression.type + BLOCK + + + io.map.index.interval + 128 + diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index 6902a3acd8736..b3daaaaac1074 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - ${sys:pulsar.log.level} - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + ${sys:pulsar.log.level} + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 568d76922bf4e..90a59b33f815a 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.1.0-SNAPSHOT .. - pulsar-io-distribution pom Pulsar :: Distribution :: IO - ${project.groupId} @@ -46,7 +43,6 @@ provided - @@ -69,9 +65,32 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - generator-connector-config @@ -151,5 +170,4 @@ - diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 5b652170fdbb5..71d5aaf69f76a 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -1,3 +1,4 @@ + - + bin dir @@ -38,48 +37,120 @@ . 644 - - - ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar - ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar - ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar - ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar - ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/openmldb/target/pulsar-io-jdbc-openmldb-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar - ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar - ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar - ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar - ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar - ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar - ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar - ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar - ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar - ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar - ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar - ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar - ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + + ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar + + + ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar + + + ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar + + + ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/openmldb/target/pulsar-io-jdbc-openmldb-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar + + + ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar + + + ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar + + + ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar + + + ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar + + + ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar + + + ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar + + + ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar + + + ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar + + + ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index d23ebec2ef26d..992cc41283841 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.1.0-SNAPSHOT .. - pulsar-offloader-distribution pom Pulsar :: Distribution :: Offloader - ${project.groupId} @@ -66,7 +63,6 @@ - @@ -90,6 +86,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/offloaders/src/assemble/offloaders.xml b/distribution/offloaders/src/assemble/offloaders.xml index 38f7eee906064..19c60f68c21d4 100644 --- a/distribution/offloaders/src/assemble/offloaders.xml +++ b/distribution/offloaders/src/assemble/offloaders.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -38,13 +37,11 @@ . 644 - ${basedir}/../../tiered-storage/jcloud/target/tiered-storage-jcloud-${project.version}.nar offloaders 644 - ${basedir}/../../tiered-storage/file-system/target/tiered-storage-file-system-${project.version}.nar offloaders diff --git a/distribution/pom.xml b/distribution/pom.xml index 36a3fa1c5835a..7be5264888b20 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - distribution pom Pulsar :: Distribution - - main @@ -47,7 +43,6 @@ shell - core-modules @@ -55,7 +50,6 @@ - @@ -65,6 +59,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 2702bace54e6a..69abf517b1e0f 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.1.0-SNAPSHOT .. - pulsar-server-distribution pom Pulsar :: Distribution :: Server - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} pulsar-docs-tools ${project.version} - ${project.groupId} pulsar-proxy ${project.version} - ${project.groupId} pulsar-broker-auth-oidc ${project.version} - ${project.groupId} pulsar-broker-auth-sasl ${project.version} - ${project.groupId} pulsar-client-auth-sasl ${project.version} - jline jline ${jline.version} - org.apache.zookeeper zookeeper-prometheus-metrics @@ -95,7 +85,6 @@ - ${project.groupId} pulsar-package-bookkeeper-storage @@ -107,13 +96,11 @@ - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - ${project.groupId} pulsar-client-tools @@ -125,75 +112,61 @@ - ${project.groupId} pulsar-testclient ${project.version} - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web - io.dropwizard.metrics metrics-core - io.dropwizard.metrics metrics-graphite - io.dropwizard.metrics metrics-jvm - org.xerial.snappy snappy-java - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.logging.log4j log4j-slf4j-impl - org.apache.bookkeeper.stats prometheus-metrics-provider - io.prometheus simpleclient_log4j2 - ${project.groupId} bouncy-castle-bc ${project.version} pkg - ${project.groupId} @@ -208,7 +181,6 @@ - ${project.groupId} pulsar-functions-worker @@ -225,7 +197,6 @@ - ${project.groupId} @@ -240,7 +211,6 @@ - ${project.groupId} @@ -253,7 +223,6 @@ - io.grpc @@ -263,13 +232,11 @@ org.bouncycastle bcpkix-jdk18on - io.perfmark perfmark-api compile - org.apache.bookkeeper.http @@ -291,7 +258,6 @@ vertx-web - @@ -332,6 +298,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml index 41ac24d0582da..f65a8e2fb7260 100644 --- a/distribution/server/src/assemble/bin.xml +++ b/distribution/server/src/assemble/bin.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -131,14 +130,11 @@ ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - org.apache.pulsar:pulsar-functions-runtime-all - + io.streamnative:pulsar-functions-runtime-all org.projectlombok:lombok - - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples *:tar.gz diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 9e3134a75e5bf..a119a3d35e655 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.1.0-SNAPSHOT .. - pulsar-shell-distribution pom Pulsar :: Distribution :: Shell - ${project.groupId} pulsar-client-tools ${project.version} - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web @@ -53,14 +48,11 @@ org.apache.logging.log4j log4j-slf4j-impl - io.prometheus simpleclient_log4j2 - - @@ -101,6 +93,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/shell/src/assemble/shell.xml b/distribution/shell/src/assemble/shell.xml index f823e0258b231..cd11bd8a1ef57 100644 --- a/distribution/shell/src/assemble/shell.xml +++ b/distribution/shell/src/assemble/shell.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -46,7 +45,6 @@ . 644 - bin ${basedir}/../../bin/pulsar-admin-common.sh @@ -76,7 +74,6 @@ ${basedir}/../../conf/log4j2.yaml - lib diff --git a/docker/pom.xml b/docker/pom.xml index afe55f0fe57f0..0f1842b1665ec 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT @@ -39,6 +38,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index e616ac132d319..f06ecaaf6c839 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images 3.1.0-SNAPSHOT @@ -29,7 +29,6 @@ pulsar-all-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components) pom - ${project.groupId} @@ -64,7 +63,6 @@ - git-commit-id-no-git @@ -177,7 +175,6 @@ - docker-push @@ -200,6 +197,5 @@ - diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 85d86cff12523..ec8cc5e2b46da 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images 3.1.0-SNAPSHOT @@ -29,7 +29,6 @@ pulsar-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version pom - ${project.groupId} @@ -46,12 +45,10 @@ - mirror://mirrors.ubuntu.com/mirrors.txt http://security.ubuntu.com/ubuntu/ - git-commit-id-no-git @@ -128,7 +125,6 @@ - docker-push @@ -151,6 +147,5 @@ - diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index dfb155c2d5a7d..0e5c03bc7998b 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - jclouds-shaded Apache Pulsar :: Jclouds shaded - - org.apache.jclouds jclouds-allblobstore @@ -66,7 +61,6 @@ javax.annotation-api - @@ -82,7 +76,6 @@ true true false - com.google.guava:guava @@ -106,7 +99,6 @@ com.google.errorprone:* - com.google @@ -140,21 +132,20 @@ com.google.errorprone org.apache.pulsar.jcloud.shade.com.google.errorprone - - - org.apache.jclouds:jclouds-core - - - lib/gson*jar - - - + + org.apache.jclouds:jclouds-core + + + lib/gson*jar + + + diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index a8cb560b7b376..3e80ee8582e30 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - managed-ledger Managed Ledger - org.apache.bookkeeper bookkeeper-server - org.apache.bookkeeper.stats prometheus-metrics-provider - org.apache.bookkeeper.stats codahale-metrics-provider ${bookkeeper.version} - com.google.protobuf protobuf-java - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - ${project.groupId} testmocks ${project.version} test - org.apache.zookeeper zookeeper @@ -99,29 +88,25 @@ - io.dropwizard.metrics - metrics-core - test + io.dropwizard.metrics + metrics-core + test - org.xerial.snappy - snappy-java - test + org.xerial.snappy + snappy-java + test - org.awaitility awaitility test - org.slf4j slf4j-api - - @@ -141,7 +126,6 @@ - org.apache.maven.plugins maven-jar-plugin diff --git a/pom.xml b/pom.xml index 7e3538dc6a154..37aeb459c02a0 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 pom @@ -28,30 +27,24 @@ apache 29 - - org.apache.pulsar + io.streamnative pulsar - 3.1.0-SNAPSHOT - Pulsar Pulsar is a distributed pub-sub messaging platform with a very flexible messaging model and an intuitive client API. https://github.com/apache/pulsar - Apache Software Foundation https://www.apache.org/ 2017 - Apache Pulsar developers https://pulsar.apache.org/ - Apache License, Version 2.0 @@ -59,47 +52,38 @@ flexible messaging model and an intuitive client API. repo - - https://github.com/apache/pulsar - scm:git:https://github.com/apache/pulsar.git - scm:git:ssh://git@github.com:apache/pulsar.git + https://github.com/streamnative/pulsar + scm:git:https://github.com/streamnative/pulsar.git + scm:git:ssh://git@github.com:streamnative/pulsar.git - GitHub Actions https://github.com/apache/pulsar/actions - Github https://github.com/apache/pulsar/issues - 17 17 ${maven.compiler.target} 8 - 3.2.0 - **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java quarantine - UTF-8 UTF-8 2023-05-03T02:53:27Z true - - - + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED @@ -116,7 +100,7 @@ flexible messaging model and an intuitive client API. false 1 true - + false ${project.build.directory} @@ -129,16 +113,16 @@ flexible messaging model and an intuitive client API. false package package - 1.21 - 4.16.2 3.8.1 1.5.0 1.10.0 - 1.1.10.1 - 4.1.12.1 + 1.1.10.1 + + 4.1.12.1 + 5.1.0 4.1.94.Final 0.0.21.Final @@ -249,11 +233,9 @@ flexible messaging model and an intuitive client API. 3.4.3 1.5.2-3 2.0.6 - 1.18.3 2.2 - 3.3.0 1.1.1 @@ -266,7 +248,6 @@ flexible messaging model and an intuitive client API. 1.5.4 5.4.0 2.33.2 - 0.6.1 3.0.0 @@ -300,20 +281,16 @@ flexible messaging model and an intuitive client API. 0.9.44 1.6.1 6.4.0 - rename-netty-native-libs.sh - - org.jline jline ${jline3.version} - org.asynchttpclient async-http-client @@ -329,45 +306,39 @@ flexible messaging model and an intuitive client API. - org.testng testng ${testng.version} - - org.yaml - * - + + org.yaml + * + - org.hamcrest hamcrest ${hamcrest.version} test - org.awaitility awaitility ${awaitility.version} test - org.mockito mockito-core ${mockito.version} - org.mockito mockito-inline ${mockito.version} - org.apache.zookeeper zookeeper @@ -437,7 +408,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-server @@ -466,13 +436,11 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper cpu-affinity ${bookkeeper.version} - io.vertx vertx-core @@ -483,38 +451,33 @@ flexible messaging model and an intuitive client API. vertx-web ${vertx.version} - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - * - - + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.zookeeper + * + + - org.apache.bookkeeper bookkeeper-common-allocator ${bookkeeper.version} - org.apache.bookkeeper bookkeeper-tools-framework ${bookkeeper.version} - org.reflections reflections ${reflections.version} - org.apache.bookkeeper @@ -543,7 +506,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper @@ -576,19 +538,16 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-common ${bookkeeper.version} - org.apache.bookkeeper.stats bookkeeper-stats-api ${bookkeeper.version} - org.apache.bookkeeper.stats datasketches-metrics-provider @@ -600,37 +559,31 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper.stats prometheus-metrics-provider ${bookkeeper.version} - org.rocksdb rocksdbjni ${rocksdb.version} - org.eclipse.jetty jetty-server ${jetty.version} - org.eclipse.jetty jetty-alpn-conscrypt-server ${jetty.version} - org.conscrypt conscrypt-openjdk-uber ${conscrypt.version} - org.eclipse.jetty jetty-bom @@ -638,7 +591,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty netty-bom @@ -646,7 +598,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty.incubator netty-incubator-transport-classes-io_uring @@ -669,79 +620,66 @@ flexible messaging model and an intuitive client API. ${netty-iouring.version} linux-aarch_64 - com.beust jcommander ${jcommander.version} - com.google.guava guava ${guava.version} - com.google.inject guice ${guice.version} - com.google.inject.extensions guice-assistedinject ${guice.version} - org.apache.commons commons-lang3 ${commons-lang3.version} - org.apache.commons commons-compress ${commons-compress.version} - commons-configuration commons-configuration ${commons-configuration.version} - commons-io commons-io ${commons-io.version} - org.apache.commons commons-text ${commons-text.version} - org.slf4j slf4j-api ${slf4j.version} - org.slf4j slf4j-simple ${slf4j.version} - org.slf4j jcl-over-slf4j ${slf4j.version} - org.apache.logging.log4j log4j-bom @@ -749,49 +687,41 @@ flexible messaging model and an intuitive client API. pom import - commons-codec commons-codec ${commons-codec.version} - org.glassfish.jersey.core jersey-server ${jersey.version} - org.glassfish.jersey.core jersey-client ${jersey.version} - org.glassfish.jersey.inject jersey-hk2 ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet ${jersey.version} - javax.ws.rs javax.ws.rs-api ${javax.ws.rs-api.version} - org.glassfish.jersey.media jersey-media-json-jackson @@ -803,19 +733,16 @@ flexible messaging model and an intuitive client API. - org.glassfish.jersey.media jersey-media-multipart ${jersey.version} - net.java.dev.jna jna ${jna.version} - com.github.docker-java docker-java-core @@ -837,7 +764,6 @@ flexible messaging model and an intuitive client API. docker-java-transport-zerodep ${docker-java.version} - com.fasterxml.jackson jackson-bom @@ -845,56 +771,46 @@ flexible messaging model and an intuitive client API. pom import - org.codehaus.jettison jettison ${jettison.version} - com.fasterxml.woodstox woodstox-core ${woodstox.version} - - org.hdrhistogram HdrHistogram ${hdrHistogram.version} - io.swagger swagger-core ${swagger.version} - io.swagger swagger-annotations ${swagger.version} - javax.servlet javax.servlet-api ${javax.servlet-api} - com.github.ben-manes.caffeine caffeine ${caffeine.version} - org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} - com.cronutils cron-utils @@ -906,19 +822,16 @@ flexible messaging model and an intuitive client API. - com.yahoo.athenz athenz-zts-java-client-core ${athenz.version} - com.yahoo.athenz athenz-zpe-java-client ${athenz.version} - com.yahoo.athenz athenz-cert-refresher @@ -930,7 +843,6 @@ flexible messaging model and an intuitive client API. - com.yahoo.athenz athenz-auth-core @@ -942,67 +854,56 @@ flexible messaging model and an intuitive client API. - com.github.zafarkhaja java-semver ${java-semver.version} - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_log4j2 ${prometheus.version} - io.prometheus simpleclient_servlet ${prometheus.version} - io.prometheus simpleclient_jetty ${prometheus.version} - io.prometheus simpleclient_caffeine ${prometheus.version} - com.carrotsearch hppc ${hppc.version} - io.etcd jetcd-core ${jetcd.version} - io.etcd jetcd-test ${jetcd.version} - org.apache.spark spark-streaming_2.10 @@ -1030,7 +931,6 @@ flexible messaging model and an intuitive client API. - io.jsonwebtoken jjwt-api @@ -1046,13 +946,11 @@ flexible messaging model and an intuitive client API. jjwt-jackson ${jsonwebtoken.version} - net.jodah typetools ${typetools.version} - io.grpc grpc-bom @@ -1060,7 +958,6 @@ flexible messaging model and an intuitive client API. pom import - io.grpc grpc-all @@ -1084,7 +981,6 @@ flexible messaging model and an intuitive client API. - io.grpc grpc-xds @@ -1096,25 +992,21 @@ flexible messaging model and an intuitive client API. - com.google.http-client google-http-client ${google-http-client.version} - com.google.http-client google-http-client-jackson2 ${google-http-client.version} - com.google.http-client google-http-client-gson ${google-http-client.version} - io.perfmark perfmark-api @@ -1127,7 +1019,6 @@ flexible messaging model and an intuitive client API. - com.google.protobuf protobuf-bom @@ -1135,19 +1026,16 @@ flexible messaging model and an intuitive client API. pom import - com.google.code.gson gson ${gson.version} - com.yahoo.datasketches sketches-core ${sketches.version} - com.amazonaws aws-java-sdk-bom @@ -1155,7 +1043,6 @@ flexible messaging model and an intuitive client API. pom import - org.apache.distributedlog distributedlog-core @@ -1168,13 +1055,11 @@ flexible messaging model and an intuitive client API. - org.apache.commons commons-collections4 ${commons.collections4.version} - com.lmax @@ -1198,103 +1083,86 @@ flexible messaging model and an intuitive client API. assertj-core ${assertj-core.version} - org.projectlombok lombok ${lombok.version} - javax.annotation javax.annotation-api ${javax.annotation-api.version} - javax.xml.bind jaxb-api ${jaxb-api} - jakarta.xml.bind jakarta.xml.bind-api ${jakarta.xml.bind.version} - com.sun.activation javax.activation ${javax.activation.version} - com.sun.activation jakarta.activation ${jakarta.activation.version} - jakarta.activation jakarta.activation-api ${jakarta.activation.version} - jakarta.validation jakarta.validation-api ${jakarta.validation.version} - io.opencensus opencensus-api ${opencensus.version} - io.opencensus opencensus-contrib-http-util ${opencensus.version} - io.opencensus opencensus-contrib-grpc-metrics ${opencensus.version} - org.opensearch.client opensearch-rest-high-level-client ${opensearch.version} - co.elastic.clients elasticsearch-java ${elasticsearch-java.version} - joda-time joda-time ${joda.version} - org.javassist javassist ${javassist.version} - net.jcip jcip-annotations ${jcip.version} - io.airlift aircompressor @@ -1306,25 +1174,21 @@ flexible messaging model and an intuitive client API. - org.objenesis objenesis ${objenesis.version} - org.apache.httpcomponents httpclient ${apache-http-client.version} - org.apache.httpcomponents httpcore ${apache-httpcomponents.version} - com.github.spotbugs spotbugs-annotations @@ -1332,31 +1196,26 @@ flexible messaging model and an intuitive client API. provided true - com.google.errorprone error_prone_annotations ${errorprone.version} - com.google.j2objc j2objc-annotations ${j2objc-annotations.version} - org.yaml snakeyaml ${snakeyaml.version} - org.apache.ant ant ${ant.version} - com.squareup.okhttp3 okhttp @@ -1377,7 +1236,6 @@ flexible messaging model and an intuitive client API. okio ${okio.version} - org.jetbrains.kotlin kotlin-stdlib @@ -1388,25 +1246,21 @@ flexible messaging model and an intuitive client API. kotlin-stdlib-common ${kotlin-stdlib.version} - org.jetbrains.kotlin kotlin-stdlib-jdk8 ${kotlin-stdlib.version} - com.github.luben zstd-jni ${zstd-jni.version} - - com.typesafe.netty - netty-reactive-streams - ${netty-reactive-streams.version} + com.typesafe.netty + netty-reactive-streams + ${netty-reactive-streams.version} - org.roaringbitmap RoaringBitmap @@ -1419,47 +1273,40 @@ flexible messaging model and an intuitive client API. - - org.apache.pulsar + io.streamnative buildtools ${project.version} test - org.testng testng test - org.mockito mockito-core test - org.mockito mockito-inline test - com.github.stefanbirkner system-lambda ${system-lambda.version} test - org.assertj assertj-core test - org.projectlombok lombok @@ -1470,7 +1317,6 @@ flexible messaging model and an intuitive client API. javax.annotation-api provided - org.apache.bookkeeper @@ -1480,8 +1326,8 @@ flexible messaging model and an intuitive client API. tests - org.bouncycastle - * + org.bouncycastle + * org.slf4j @@ -1501,8 +1347,12 @@ flexible messaging model and an intuitive client API. + + org.apache.logging.log4j + log4j-layout-template-json + 2.20.0 + - ${project.artifactId} @@ -1577,7 +1427,6 @@ flexible messaging model and an intuitive client API. - org.commonjava.maven.plugins directory-maven-plugin @@ -1592,14 +1441,13 @@ flexible messaging model and an intuitive client API. pulsar.basedir - org.apache.pulsar + io.streamnative pulsar - pl.project13.maven git-commit-id-plugin @@ -1624,7 +1472,6 @@ flexible messaging model and an intuitive client API. - com.mycila license-maven-plugin @@ -1725,33 +1572,26 @@ flexible messaging model and an intuitive client API. src/assemble/README.bin.txt src/assemble/LICENSE.bin.txt src/assemble/NOTICE.bin.txt - src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java bin/proto/MLDataFormats_pb2.py - **/avro/generated/*.java - **/*.avsc - src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionKey.java src/main/java/org/apache/pulsar/io/kinesis/fbs/KeyValue.java src/main/java/org/apache/pulsar/io/kinesis/fbs/Message.java - src/main/java/org/apache/bookkeeper/mledger/util/AbstractCASReferenceCounted.java - dependency-reduced-pom.xml - pulsar-client-go/go.mod pulsar-client-go/go.sum @@ -1759,15 +1599,12 @@ flexible messaging model and an intuitive client API. pulsar-function-go/go.sum pulsar-function-go/examples/go.mod pulsar-function-go/examples/go.sum - **/META-INF/services/com.scurrilous.circe.HashProvider **/META-INF/services/io.trino.spi.Plugin - **/django/stats/migrations/*.py **/conf/uwsgi_params - **/*.crt **/*.key @@ -1780,21 +1617,16 @@ flexible messaging model and an intuitive client API. certificate-authority/index.txt certificate-authority/serial certificate-authority/README.md - **/zk-3.5-test-data/* - **/requirements.txt - conf/schema_example.conf **/templates/*.tpl - **/.helmignore **/_helpers.tpl - **/*.md .github/** @@ -1823,6 +1655,7 @@ flexible messaging model and an intuitive client API. **/*.so.* **/*.dylib src/test/resources/*.txt + **/dependency-reduced-pom.xml @@ -1831,55 +1664,106 @@ flexible messaging model and an intuitive client API. maven-enforcer-plugin ${maven-enforcer-plugin.version} - - enforce-maven - - enforce - - - - - 17 - Java 17+ is required to build Pulsar. - - - 3.6.1 - - - - + + enforce-maven + + enforce + + + + + 17 + Java 17+ is required to build Pulsar. + + + 3.6.1 + + + + - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - false - - - source-release-assembly-tar-gz - generate-sources - - single - - - ${skipSourceReleaseAssembly} - true - - src/assembly-source-package.xml - - apache-pulsar-${project.version}-src - false - - tar.gz - - posix - - - + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + false + + + source-release-assembly-tar-gz + generate-sources + + single + + + ${skipSourceReleaseAssembly} + true + + src/assembly-source-package.xml + + apache-pulsar-${project.version}-src + false + + tar.gz + + posix + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + true + + ossrh + https://s01.oss.sonatype.org/ + 60285aee9d4161 + true + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + none + false + + + + + + org.apache.maven.plugins + maven-release-plugin + 3.0.0-M1 + + true + false + release + deploy + - @@ -1900,10 +1784,10 @@ flexible messaging model and an intuitive client API. maven-surefire-plugin - ${include} + ${include} - **/*$*,${exclude} + **/*$*,${exclude} ${groups} ${excludedGroups} @@ -1987,13 +1871,13 @@ flexible messaging model and an intuitive client API. com.github.spotbugs spotbugs-maven-plugin ${spotbugs-maven-plugin.version} - - - com.github.spotbugs - spotbugs - ${spotbugs.version} - - + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + org.codehaus.mojo @@ -2005,6 +1889,16 @@ flexible messaging model and an intuitive client API. docker-maven-plugin ${docker-maven.version} + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + @@ -2020,7 +1914,6 @@ flexible messaging model and an intuitive client API. - @@ -2129,7 +2022,6 @@ flexible messaging model and an intuitive client API. tests - contrib-check - - - - org.apache.rat - apache-rat-plugin - - - - check - - verify - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - check-style - verify - - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml - UTF-8 - **/proto/* - - - check - - - - - - + + + + org.apache.rat + apache-rat-plugin + + + + check + + verify + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + check-style + verify + + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml + UTF-8 + **/proto/* + + + check + + + + + + - windows @@ -2185,7 +2076,6 @@ flexible messaging model and an intuitive client API. rename-netty-native-libs.cmd - @@ -2230,37 +2120,27 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - structured-event-log - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - pulsar-metadata jclouds-shaded - pulsar-package-management - distribution docker tests - core-modules @@ -2290,31 +2170,23 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - distribution pulsar-metadata - pulsar-package-management - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-broker-auth-athenz jar Athenz authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} testmocks ${project.version} test - com.yahoo.athenz athenz-zpe-java-client - org.bouncycastle bcpkix-jdk18on - - @@ -79,7 +69,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -97,7 +86,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -111,7 +99,6 @@ - diff --git a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 157d8d0815d4f..e898923dfd659 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-broker-auth-oidc jar Open ID Connect authentication plugin for broker - 0.11.5 - - ${project.groupId} pulsar-broker-common @@ -50,29 +44,24 @@ - com.auth0 java-jwt 4.3.0 - com.auth0 jwks-rsa 0.22.0 - com.github.ben-manes.caffeine caffeine - org.asynchttpclient async-http-client - io.kubernetes client-java @@ -97,7 +86,6 @@ - io.jsonwebtoken jjwt-api @@ -110,16 +98,13 @@ ${jsonwebtoken.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - @@ -141,8 +126,6 @@ - - @@ -162,7 +145,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 2957159ca93e8..cd06ee0f6f16e 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-broker-auth-sasl jar SASL authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - org.apache.kerby kerby-config @@ -53,7 +47,6 @@ - org.apache.kerby kerb-simplekdc @@ -66,30 +59,25 @@ - ${project.groupId} pulsar-proxy ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-client-auth-sasl ${project.version} test - - @@ -111,8 +99,6 @@ - - @@ -132,7 +118,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index b81f2b7621ba7..a6d0e21a0ca59 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-broker-common Common classes used in multiple broker modules - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - io.prometheus simpleclient_jetty - javax.servlet javax.servlet-api - javax.ws.rs javax.ws.rs-api - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.bouncycastle @@ -76,14 +65,12 @@ ${bouncycastle.bc-fips.version} test - org.awaitility awaitility test - @@ -103,7 +90,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -117,7 +103,6 @@ - maven-resources-plugin diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 4c39d7c4b79d0..645253a41ffed 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-broker jar Pulsar Broker - commons-codec commons-codec - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - org.slf4j slf4j-api - io.netty netty-transport - com.google.protobuf protobuf-java - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - ${project.groupId} managed-ledger ${project.version} - org.apache.curator curator-recipes - org.apache.bookkeeper stream-storage-server @@ -119,170 +105,139 @@ - org.apache.bookkeeper bookkeeper-tools-framework - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-transaction-common ${project.version} - ${project.groupId} pulsar-io-batch-discovery-triggerers ${project.version} test - ${project.groupId} testmocks ${project.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - io.dropwizard.metrics - metrics-core + io.dropwizard.metrics + metrics-core - - org.xerial.snappy - snappy-java + org.xerial.snappy + snappy-java - - ${project.groupId} pulsar-functions-worker ${project.version} - ${project.groupId} pulsar-functions-local-runner-original ${project.version} test - ${project.groupId} pulsar-client-messagecrypto-bc ${project.version} - org.awaitility awaitility test - - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.media jersey-media-json-jackson - org.glassfish.jersey.test-framework jersey-test-framework-core test ${jersey.version} - org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-grizzly2 test ${jersey.version} - jakarta.activation jakarta.activation-api - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - com.fasterxml.jackson.module jackson-module-jsonSchema - org.slf4j jcl-over-slf4j - com.google.guava guava - ${project.groupId} pulsar-docs-tools @@ -294,80 +249,65 @@ - com.beust jcommander - io.swagger swagger-annotations provided - io.prometheus simpleclient - io.prometheus simpleclient_jetty - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_caffeine - io.swagger swagger-core provided - org.hdrhistogram HdrHistogram - com.google.code.gson gson - com.github.zafarkhaja java-semver - org.apache.avro avro ${avro.version} - com.carrotsearch hppc - org.roaringbitmap RoaringBitmap - com.github.oshi oshi-core-java11 - ${project.groupId} pulsar-functions-api-examples @@ -375,7 +315,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples-builtin @@ -383,7 +322,6 @@ pom test - ${project.groupId} pulsar-io-batch-data-generator @@ -391,7 +329,6 @@ pom test - ${project.groupId} pulsar-io-data-generator @@ -399,7 +336,6 @@ pom test - ${project.groupId} pulsar-metadata @@ -407,7 +343,6 @@ test-jar test - javax.xml.bind jaxb-api @@ -418,42 +353,33 @@ - com.sun.activation javax.activation - - ${project.groupId} pulsar-transaction-coordinator ${project.version} - - ${project.groupId} pulsar-package-core ${project.version} - io.etcd jetcd-test test - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - - @@ -476,7 +402,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -507,7 +432,6 @@ - maven-dependency-plugin @@ -560,7 +484,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -576,7 +499,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin @@ -645,7 +567,6 @@ - maven-resources-plugin @@ -676,7 +597,6 @@ - @@ -695,7 +615,6 @@ test-jar test - ${project.groupId} pulsar-package-core @@ -703,7 +622,6 @@ test-jar test - ${project.groupId} pulsar-metadata diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 41de4d2e06bde..b847e5f53cb85 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-1x-base Pulsar Client 1.x Compatibility Base pom - pulsar-client-2x-shaded pulsar-client-1x - @@ -61,7 +57,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -77,5 +72,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 307a1d76f7490..95e902fdf770c 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base 3.1.0-SNAPSHOT .. - pulsar-client-1x Pulsar Client 1.x Compatibility API - ${project.groupId} pulsar-client-2x-shaded ${project.version} - com.google.guava guava - org.apache.commons commons-lang3 - - @@ -70,7 +63,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -90,5 +82,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml b/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml index 7938e60bf4330..d191e191d52ca 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base 3.1.0-SNAPSHOT .. - pulsar-client-2x-shaded Pulsar Client 2.x Shaded API - ${project.groupId} @@ -39,10 +36,9 @@ ${project.version} - - + org.apache.maven.plugins maven-shade-plugin @@ -56,16 +52,15 @@ true true false - - org.apache.pulsar:pulsar-client - org.apache.pulsar:pulsar-client-api + io.streamnative:pulsar-client + io.streamnative:pulsar-client-api - org.apache.pulsar:pulsar-client + io.streamnative:pulsar-client ** @@ -93,6 +88,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index be666085c2c5f..1684e0479128a 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - .. - - - pulsar-client-admin-api - Pulsar Client Admin :: API - - - - ${project.groupId} - pulsar-client-api - ${project.version} - - - - org.slf4j - slf4j-api - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 3.1.0-SNAPSHOT + .. + + pulsar-client-admin-api + Pulsar Client Admin :: API + + + ${project.groupId} + pulsar-client-api + ${project.version} + + + org.slf4j + slf4j-api + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 4aaefe3a275b5..5e536ceebb75d 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-admin Pulsar Client Admin - ${project.groupId} @@ -74,7 +70,6 @@ - maven-antrun-plugin @@ -87,15 +82,12 @@ - + - org.apache.maven.plugins maven-shade-plugin @@ -108,11 +100,10 @@ true true - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -126,7 +117,7 @@ com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -148,7 +139,7 @@ org.yaml:snakeyaml io.swagger:* - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -156,7 +147,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -166,7 +157,7 @@ - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-admin-original ** @@ -177,7 +168,7 @@ - + org.asynchttpclient org.apache.pulsar.shade.org.asynchttpclient @@ -259,51 +250,75 @@ org.reactivestreams org.apache.pulsar.shade.org.reactivestreams - - io.grpc - org.apache.pulsar.shade.io.grpc - - - okio - org.apache.pulsar.shade.okio - - - com.squareup - org.apache.pulsar.shade.com.squareup - - - io.opencensus - org.apache.pulsar.shade.io.opencensus - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - org.objenesis - org.apache.pulsar.shade.org.objenesis - - - org.yaml - org.apache.pulsar.shade.org.yaml - - - io.swagger - org.apache.pulsar.shade.io.swagger - - - org.apache.bookkeeper - org.apache.pulsar.shade.org.apache.bookkeeper - + + io.grpc + org.apache.pulsar.shade.io.grpc + + + okio + org.apache.pulsar.shade.okio + + + com.squareup + org.apache.pulsar.shade.com.squareup + + + io.opencensus + org.apache.pulsar.shade.io.opencensus + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + org.objenesis + org.apache.pulsar.shade.org.objenesis + + + org.yaml + org.apache.pulsar.shade.org.yaml + + + io.swagger + org.apache.pulsar.shade.io.swagger + + + org.apache.bookkeeper + org.apache.pulsar.shade.org.apache.bookkeeper + - - + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index c811236705971..9ddf04ae917fe 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-admin-original Pulsar Client Admin Original - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - org.glassfish.jersey.core jersey-client - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.media jersey-media-multipart - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api @@ -91,30 +79,25 @@ javax.activation runtime - com.google.guava guava - com.google.code.gson gson - ${project.groupId} pulsar-package-core ${project.version} - org.hamcrest hamcrest test - @@ -141,7 +124,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index c893bb6f7e9e3..603dc526df90d 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-all Pulsar Client All - ${project.groupId} @@ -70,7 +67,6 @@ test - @@ -108,7 +104,6 @@ - maven-antrun-plugin @@ -121,15 +116,12 @@ - + - org.apache.maven.plugins @@ -146,11 +138,10 @@ true true false - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -176,8 +167,7 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -205,7 +195,7 @@ org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -213,7 +203,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -315,24 +305,24 @@ org.apache.pulsar.shade.org.glassfish - io.grpc - org.apache.pulsar.shade.io.grpc + io.grpc + org.apache.pulsar.shade.io.grpc - okio - org.apache.pulsar.shade.okio + okio + org.apache.pulsar.shade.okio - com.squareup - org.apache.pulsar.shade.com.squareup + com.squareup + org.apache.pulsar.shade.com.squareup - io.opencensus - org.apache.pulsar.shade.io.opencensus + io.opencensus + org.apache.pulsar.shade.io.opencensus - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty org.objenesis @@ -389,14 +379,13 @@ - - + + - - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - .. - - - pulsar-client-api - Pulsar Client :: API - - - - - - com.google.protobuf - protobuf-java - provided - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + com.google.protobuf + protobuf-java + provided + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-api/src/main/resources/findbugsExclude.xml b/pulsar-client-api/src/main/resources/findbugsExclude.xml index 9d73ac29a7bdb..353f01a7bb285 100644 --- a/pulsar-client-api/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-api/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 3cfcc8be49c98..e676e6347524c 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-auth-athenz jar Athenz authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.yahoo.athenz athenz-zts-java-client-core - com.yahoo.athenz athenz-cert-refresher - org.bouncycastle bcpkix-jdk18on - com.google.guava guava - org.apache.commons commons-lang3 - @@ -77,7 +67,6 @@ ${pulsar.client.compiler.release} - org.apache.maven.plugins maven-enforcer-plugin @@ -108,7 +97,6 @@ - org.gaul modernizer-maven-plugin @@ -126,7 +114,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -144,7 +131,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-auth-sasl jar SASL authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.google.guava guava - org.apache.commons commons-lang3 - org.projectlombok lombok - javax.ws.rs javax.ws.rs-api - org.glassfish.jersey.core jersey-client - - @@ -78,7 +67,6 @@ ${pulsar.client.compiler.release} - org.apache.maven.plugins maven-enforcer-plugin @@ -109,7 +97,6 @@ - org.gaul modernizer-maven-plugin @@ -127,7 +114,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 4e2604f33169a..f95f3b9d6b011 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-messagecrypto-bc jar Message crypto for End to End encryption with BouncyCastleProvider - ${project.groupId} @@ -40,7 +37,6 @@ ${project.parent.version} provided - ${project.groupId} bouncy-castle-bc @@ -48,9 +44,7 @@ pkg true - - diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 242fc42b2954f..a046f20c98ac8 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client Pulsar Client Java - ${project.groupId} @@ -49,11 +46,8 @@ ${project.version} - - - org.apache.maven.plugins maven-dependency-plugin @@ -89,7 +83,6 @@ - maven-antrun-plugin @@ -102,15 +95,12 @@ - + - org.apache.maven.plugins @@ -125,10 +115,9 @@ true true false - - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original org.apache.bookkeeper:* org.apache.commons:commons-lang3 commons-codec:commons-codec @@ -154,19 +143,17 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common com.yahoo.datasketches:sketches-core org.objenesis:* org.yaml:snakeyaml - org.apache.avro:* com.thoughtworks.paranamer:paranamer org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -174,7 +161,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -302,14 +289,13 @@ - - + + - com.github.spotbugs spotbugs-maven-plugin @@ -323,7 +309,6 @@ - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-tools-api Pulsar Client Tools API Pulsar Client Tools API - ${project.groupId} @@ -66,7 +64,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -82,5 +79,4 @@ - diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index a3a3de19202c2..5789d8f37bb0e 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -1,3 +1,4 @@ + - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. @@ -31,7 +32,7 @@ Pulsar CLI Custom command example - org.apache.pulsar + io.streamnative pulsar-client-tools-api ${project.version} provided diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 74278374da518..b8819a0d3063d 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-tools-test Pulsar Client Tools Test Pulsar Client Tools Test - ${project.groupId} @@ -89,7 +87,6 @@ - org.apache.maven.plugins maven-deploy-plugin @@ -126,8 +123,8 @@ copy filters - - + + diff --git a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-tools Pulsar Client Tools Pulsar Client Tools - com.beust @@ -109,7 +107,6 @@ swagger-core provided - ${project.groupId} @@ -117,9 +114,8 @@ ${project.version} test - - org.apache.pulsar + io.streamnative pulsar-io-batch-discovery-triggerers ${project.version} test @@ -137,9 +133,7 @@ org.apache.commons commons-text - - @@ -159,7 +153,6 @@ - @@ -179,7 +172,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -195,5 +187,4 @@ - diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 0e45e73636612..738fbd5898e55 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-client-original Pulsar Client Java - ${project.groupId} pulsar-client-api ${project.parent.version} - ${project.groupId} pulsar-common ${project.parent.version} - ${project.groupId} bouncy-castle-bc ${project.parent.version} pkg - ${project.groupId} @@ -59,7 +53,6 @@ ${project.parent.version} true - io.netty netty-codec-http @@ -72,13 +65,11 @@ io.netty netty-codec-socks - io.swagger swagger-annotations provided - io.netty netty-resolver-dns @@ -93,50 +84,40 @@ netty-resolver-dns-native-macos osx-x86_64 - org.apache.commons commons-lang3 - org.asynchttpclient async-http-client - com.typesafe.netty netty-reactive-streams - org.slf4j slf4j-api - commons-codec commons-codec - com.yahoo.datasketches sketches-core - com.google.code.gson gson - - org.apache.avro avro ${avro.version} - org.apache.avro avro-protobuf @@ -148,36 +129,30 @@ - joda-time joda-time provided - com.google.protobuf protobuf-java provided - com.fasterxml.jackson.module jackson-module-jsonSchema - net.jcip jcip-annotations - com.github.spotbugs spotbugs-annotations provided true - ${project.groupId} @@ -185,22 +160,18 @@ ${project.parent.version} test - org.skyscreamer jsonassert ${skyscreamer.version} test - org.awaitility awaitility test - - @@ -234,7 +205,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -248,7 +218,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml index 67c012ad30b8f..1412601fec937 100644 --- a/pulsar-client/src/main/resources/findbugsExclude.xml +++ b/pulsar-client/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 8c99638878f53..a9f2492f097f9 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-common Pulsar Common Common libraries needed by client/broker/tools - ${project.groupId} pulsar-client-api ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - io.swagger swagger-annotations provided - org.slf4j slf4j-api - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.module jackson-module-parameter-names - com.fasterxml.jackson.datatype jackson-datatype-jsr310 - com.fasterxml.jackson.datatype jackson-datatype-jdk8 - com.google.guava guava - io.netty netty-handler - io.netty netty-resolver-dns - io.netty netty-transport-native-epoll linux-x86_64 - io.netty netty-transport-native-unix-common linux-x86_64 - org.apache.bookkeeper bookkeeper-common-allocator @@ -115,17 +98,14 @@ - org.apache.bookkeeper cpu-affinity - io.airlift aircompressor - org.apache.bookkeeper circe-checksum @@ -141,66 +121,54 @@ - io.netty netty-tcnative-boringssl-static - io.netty.incubator netty-incubator-transport-classes-io_uring - io.netty.incubator netty-incubator-transport-native-io_uring linux-x86_64 - io.netty.incubator netty-incubator-transport-native-io_uring linux-aarch_64 - io.netty netty-codec-haproxy - org.apache.commons commons-lang3 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - javax.ws.rs javax.ws.rs-api - commons-io commons-io - com.github.spotbugs spotbugs-annotations provided true - com.google.protobuf - protobuf-java + protobuf-java - org.bouncycastle @@ -208,38 +176,32 @@ ${bouncycastle.bc-fips.version} test - org.lz4 lz4-java 1.5.0 test - com.github.luben zstd-jni test - org.xerial.snappy snappy-java test - com.google.code.gson gson - org.awaitility awaitility test - @@ -249,7 +211,6 @@ ${pulsar.client.compiler.release} - org.gaul modernizer-maven-plugin @@ -267,7 +228,6 @@ - com.github.splunk.lightproto lightproto-maven-plugin @@ -280,7 +240,6 @@ - pl.project13.maven git-commit-id-plugin @@ -305,7 +264,6 @@ - org.codehaus.mojo templating-maven-plugin diff --git a/pulsar-common/src/main/resources/findbugsExclude.xml b/pulsar-common/src/main/resources/findbugsExclude.xml index df161c4b621a7..8216948c12408 100644 --- a/pulsar-common/src/main/resources/findbugsExclude.xml +++ b/pulsar-common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 9c7ab1061b70b..2a84bfd830350 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - .. - - - pulsar-config-validation - Annotation based config validation for Pulsar - - - - org.slf4j - slf4j-api - - - - - - - org.gaul - modernizer-maven-plugin - - true - 17 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - \ No newline at end of file + + 4.0.0 + + io.streamnative + pulsar + 3.1.0-SNAPSHOT + .. + + pulsar-config-validation + Annotation based config validation for Pulsar + + + org.slf4j + slf4j-api + + + + + + org.gaul + modernizer-maven-plugin + + true + 17 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + diff --git a/pulsar-docs-tools/pom.xml b/pulsar-docs-tools/pom.xml index 9074ad0d6c591..b1e631d994d02 100644 --- a/pulsar-docs-tools/pom.xml +++ b/pulsar-docs-tools/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - - - pulsar-docs-tools - Pulsar Documentation Generators - - - - io.swagger - swagger-annotations - - - io.swagger - swagger-core - - - com.beust - jcommander - - - + + 4.0.0 + + io.streamnative + pulsar + 3.1.0-SNAPSHOT + + pulsar-docs-tools + Pulsar Documentation Generators + + + io.swagger + swagger-annotations + + + io.swagger + swagger-core + + + com.beust + jcommander + + diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 7735a0b142745..f0bc56e948bc2 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-api Pulsar Functions :: API - - org.slf4j slf4j-api - net.jodah typetools @@ -48,16 +44,13 @@ ${project.version} compile - ${project.groupId} pulsar-client-admin-api ${project.version} compile - - @@ -81,16 +74,15 @@ org.apache.maven.plugins maven-checkstyle-plugin - - checkstyle - verify - - check - - + + checkstyle + verify + + check + + - diff --git a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml index 9638cfcca8da9..b74af4c8ceb2a 100644 --- a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-instance Pulsar Functions :: Instance - - org.apache.logging.log4j @@ -46,55 +42,46 @@ org.apache.logging.log4j log4j-core - ${project.groupId} pulsar-functions-utils ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - - + ${project.groupId} pulsar-client-admin-original ${project.version} - - + ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-messagecrypto-bc ${project.parent.version} - org.apache.bookkeeper stream-storage-java-client @@ -109,24 +96,20 @@ - io.grpc grpc-stub - io.grpc grpc-all - io.perfmark perfmark-api runtime - org.apache.bookkeeper bookkeeper-common @@ -141,61 +124,50 @@ - com.yahoo.datasketches sketches-core - com.google.guava guava - com.beust jcommander - net.jodah typetools - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_httpserver ${prometheus.version} - io.prometheus.jmx collector ${prometheus-jmx.version} - org.awaitility awaitility test - - @@ -215,7 +187,6 @@ - @@ -271,5 +242,4 @@ - diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml index 7fe247d2ab20a..4c04f27f66fbe 100644 --- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-api-examples-builtin Pulsar Functions :: API Examples (NAR) - ${project.groupId} @@ -37,14 +35,36 @@ ${project.version} - org.apache.nifi nifi-nar-maven-plugin + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index b383295423f40..93afe47845908 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-api-examples Pulsar Functions :: API Examples - org.slf4j slf4j-api - ${project.groupId} pulsar-functions-api @@ -52,7 +49,6 @@ ${project.version} - @@ -87,5 +83,4 @@ - diff --git a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 37e9d40249d49..10e3d1f6d6d37 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.1.0-SNAPSHOT - .. - - - pulsar-functions-local-runner - Pulsar Functions :: Local Runner Shaded - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - org.asynchttpclient - async-http-client - ${asynchttpclient.version} - jar - true - org/asynchttpclient/config/ahc-default.properties - ${project.build.directory}/classes - - - - - - - - - maven-antrun-plugin - - - shade-ahc-properties - prepare-package - - run - - - - - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - false - - - - - - - org.apache.pulsar:* - org.apache.bookkeeper:* - commons-*:* - org.apache.commons:* - com.fasterxml.jackson.*:* - io.netty:* - com.google.*:* - javax.servlet:* - org.reactivestreams:reactive-streams - org.apache.commons:* - io.swagger:* - org.yaml:snakeyaml - io.perfmark:* - io.prometheus:* - io.prometheus.jmx:* - javax.ws.rs:* - org.tukaani:xz - com.github.zafarkhaja:java-semver - net.java.dev.jna:* - org.apache.zookeeper:* - com.thoughtworks.paranamer:paranamer - jline:* - org.rocksdb:* - org.eclipse.jetty*:* - org.apache.avro:avro - com.beust:* - net.jodah:* - io.airlift:* - com.yahoo.datasketches:* - io.netty.resolver:* - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + false + + + + + + + io.streamnative:* + org.apache.bookkeeper:* + commons-*:* + org.apache.commons:* + com.fasterxml.jackson.*:* + io.netty:* + com.google.*:* + javax.servlet:* + org.reactivestreams:reactive-streams + org.apache.commons:* + io.swagger:* + org.yaml:snakeyaml + io.perfmark:* + io.prometheus:* + io.prometheus.jmx:* + javax.ws.rs:* + org.tukaani:xz + com.github.zafarkhaja:java-semver + net.java.dev.jna:* + org.apache.zookeeper:* + com.thoughtworks.paranamer:paranamer + jline:* + org.rocksdb:* + org.eclipse.jetty*:* + org.apache.avro:avro + com.beust:* + net.jodah:* + io.airlift:* + com.yahoo.datasketches:* + io.netty.resolver:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + - - com.google - org.apache.pulsar.functions.runtime.shaded.com.google - - - org.apache.jute - org.apache.pulsar.functions.runtime.shaded.org.apache.jute - - - javax.servlet - org.apache.pulsar.functions.runtime.shaded.javax.servlet - - - net.jodah - org.apache.pulsar.functions.runtime.shaded.net.jodah - - - org.lz4 - org.apache.pulsar.functions.runtime.shaded.org.lz4 - - - org.reactivestreams - org.apache.pulsar.functions.runtime.shaded.org.reactivestreams - - - org.apache.commons - org.apache.pulsar.functions.runtime.shaded.org.apache.commons - - - io.swagger - org.apache.pulsar.functions.runtime.shaded.io.swagger - - - org.yaml - org.apache.pulsar.functions.runtime.shaded.org.yaml - - - org.jctools - org.apache.pulsar.functions.runtime.shaded.org.jctools - - - com.squareup.okhttp - org.apache.pulsar.functions.runtime.shaded.com.squareup.okhttp - - - io.grpc - org.apache.pulsar.functions.runtime.shaded.io.grpc - - - io.perfmark - org.apache.pulsar.functions.runtime.shaded.io.perfmark - - - org.joda - org.apache.pulsar.functions.runtime.shaded.org.joda - - - javax.ws.rs - org.apache.pulsar.functions.runtime.shaded.javax.ws.rs - - - io.kubernetes - org.apache.pulsar.functions.runtime.shaded.io.kubernetes - - - io.opencensus - org.apache.pulsar.functions.runtime.shaded.io.opencensus - - - net.jpountz - org.apache.pulsar.functions.runtime.shaded.net.jpountz - - - commons-configuration - org.apache.pulsar.functions.runtime.shaded.commons-configuration - - - org.tukaani - org.apache.pulsar.functions.runtime.shaded.org.tukaani - - - com.github - org.apache.pulsar.functions.runtime.shaded.com.github - - - commons-io - org.apache.pulsar.functions.runtime.shaded.commons-io - - - org.apache.distributedlog - org.apache.pulsar.functions.runtime.shaded.org.apache.distributedlog - - - - com.fasterxml - org.apache.pulsar.functions.runtime.shaded.com.fasterxml - - - org.inferred - org.apache.pulsar.functions.runtime.shaded.org.inferred - - - org.apache.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - - - org.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.bookkeeper - - - dlshade - org.apache.pulsar.functions.runtime.shaded.dlshade - - - - org.codehaus.jackson - org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson - - - net.java.dev.jna - org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna - - - org.apache.curator - org.apache.pulsar.functions.runtime.shaded.org.apache.curator - - - javax.validation - org.apache.pulsar.functions.runtime.shaded.javax.validation - - - javax.activation - org.apache.pulsar.functions.runtime.shaded.javax.activation - - - io.prometheus - org.apache.pulsar.functions.runtime.shaded.io.prometheus - - - io.prometheus.jmx - org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx - - - org.apache.zookeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper - - - io.jsonwebtoken - org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken - - - commons-codec - org.apache.pulsar.functions.runtime.shaded.commons-codec - - - com.thoughtworks.paranamer - org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer - - - org.codehaus.mojo - org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo - - - com.github.luben - org.apache.pulsar.functions.runtime.shaded.com.github.luben - - - jline - org.apache.pulsar.functions.runtime.shaded.jline - - - org.xerial.snappy - org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy - - - javax.annotation - org.apache.pulsar.functions.runtime.shaded.javax.annotation - - - org.checkerframework - org.apache.pulsar.functions.runtime.shaded.org.checkerframework - - - org.apache.yetus - org.apache.pulsar.functions.runtime.shaded.org.apache.yetus - - - commons-cli - org.apache.pulsar.functions.runtime.shaded.commons-cli - - - commons-lang - org.apache.pulsar.functions.runtime.shaded.commons-lang - - - com.squareup.okio - org.apache.pulsar.functions.runtime.shaded.com.squareup.okio - - - org.rocksdb - org.apache.pulsar.functions.runtime.shaded.org.rocksdb - - - org.objenesis - org.apache.pulsar.functions.runtime.shaded.org.objenesis - - - org.eclipse.jetty - org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty - - - org.apache.avro - org.apache.pulsar.functions.runtime.shaded.org.apache.avro - - - avro.shaded - org.apache.pulsar.functions.runtime.shaded.avo.shaded - - - com.yahoo.datasketches - org.apache.pulsar.shaded.com.yahoo.datasketches - - - com.yahoo.sketches - org.apache.pulsar.shaded.com.yahoo.sketches - - - com.beust - org.apache.pulsar.functions.runtime.shaded.com.beust - - - - io.netty - org.apache.pulsar.functions.runtime.shaded.io.netty - - - org.hamcrest - org.apache.pulsar.functions.runtime.shaded.org.hamcrest - - - aj.org - org.apache.pulsar.functions.runtime.shaded.aj.org - - - com.scurrilous - org.apache.pulsar.functions.runtime.shaded.com.scurrilous - - - okio - org.apache.pulsar.functions.runtime.shaded.okio - - + + com.fasterxml + org.apache.pulsar.functions.runtime.shaded.com.fasterxml + + + org.inferred + org.apache.pulsar.functions.runtime.shaded.org.inferred + + + org.apache.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + + + org.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.bookkeeper + + + dlshade + org.apache.pulsar.functions.runtime.shaded.dlshade + + + + org.codehaus.jackson + org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson + + + net.java.dev.jna + org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna + + + org.apache.curator + org.apache.pulsar.functions.runtime.shaded.org.apache.curator + + + javax.validation + org.apache.pulsar.functions.runtime.shaded.javax.validation + + + javax.activation + org.apache.pulsar.functions.runtime.shaded.javax.activation + + + io.prometheus + org.apache.pulsar.functions.runtime.shaded.io.prometheus + + + io.prometheus.jmx + org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx + + + org.apache.zookeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper + + + io.jsonwebtoken + org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken + + + commons-codec + org.apache.pulsar.functions.runtime.shaded.commons-codec + + + com.thoughtworks.paranamer + org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer + + + org.codehaus.mojo + org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo + + + com.github.luben + org.apache.pulsar.functions.runtime.shaded.com.github.luben + + + jline + org.apache.pulsar.functions.runtime.shaded.jline + + + org.xerial.snappy + org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy + + + javax.annotation + org.apache.pulsar.functions.runtime.shaded.javax.annotation + + + org.checkerframework + org.apache.pulsar.functions.runtime.shaded.org.checkerframework + + + org.apache.yetus + org.apache.pulsar.functions.runtime.shaded.org.apache.yetus + + + commons-cli + org.apache.pulsar.functions.runtime.shaded.commons-cli + + + commons-lang + org.apache.pulsar.functions.runtime.shaded.commons-lang + + + com.squareup.okio + org.apache.pulsar.functions.runtime.shaded.com.squareup.okio + + + org.rocksdb + org.apache.pulsar.functions.runtime.shaded.org.rocksdb + + + org.objenesis + org.apache.pulsar.functions.runtime.shaded.org.objenesis + + + org.eclipse.jetty + org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty + + + org.apache.avro + org.apache.pulsar.functions.runtime.shaded.org.apache.avro + + + avro.shaded + org.apache.pulsar.functions.runtime.shaded.avo.shaded + + + com.yahoo.datasketches + org.apache.pulsar.shaded.com.yahoo.datasketches + + + com.yahoo.sketches + org.apache.pulsar.shaded.com.yahoo.sketches + + + com.beust + org.apache.pulsar.functions.runtime.shaded.com.beust + + + + io.netty + org.apache.pulsar.functions.runtime.shaded.io.netty + + + org.hamcrest + org.apache.pulsar.functions.runtime.shaded.org.hamcrest + + + aj.org + org.apache.pulsar.functions.runtime.shaded.aj.org + + + com.scurrilous + org.apache.pulsar.functions.runtime.shaded.com.scurrilous + + + okio + org.apache.pulsar.functions.runtime.shaded.okio + + - - org.asynchttpclient - org.apache.pulsar.functions.runtime.shaded.org.asynchttpclient - - - io.airlift - org.apache.pulsar.functions.runtime.shaded.io.airlift - - - - - - - - - + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 5779695eae8e4..ac0364bf97d91 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.1.0-SNAPSHOT - .. - - - pulsar-functions-local-runner-original - Pulsar Functions :: Local Runner original - - - - ${project.groupId} - pulsar-functions-runtime - ${project.parent.version} - - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - io.grpc - grpc-all - - - - io.perfmark - perfmark-api - runtime - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - check - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 3.1.0-SNAPSHOT + .. + + pulsar-functions-local-runner-original + Pulsar Functions :: Local Runner original + + + ${project.groupId} + pulsar-functions-runtime + ${project.parent.version} + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + io.grpc + grpc-all + + + io.perfmark + perfmark-api + runtime + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + check + verify + + check + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml index ad183eb5d9427..c7c41fe9ea252 100644 --- a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 1d9973a4d41a8..c72e15ac1421f 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-functions Pulsar Functions :: Parent - main @@ -41,24 +39,23 @@ !true - - proto - api-java - java-examples - java-examples-builtin - utils - instance - runtime - runtime-all - worker - secrets - localrun - localrun-shaded - + + proto + api-java + java-examples + java-examples-builtin + utils + instance + runtime + runtime-all + worker + secrets + localrun + localrun-shaded + - - - core-modules + + core-modules proto api-java @@ -72,8 +69,6 @@ secrets localrun - - + - diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index b84b2d9181a38..c56e4bbf833dd 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.1.0-SNAPSHOT - - - pulsar-functions-proto - Pulsar Functions :: Proto - - - - - com.google.protobuf - protobuf-java - - - - com.google.protobuf - protobuf-java-util - - - - javax.annotation - javax.annotation-api - - - - io.grpc - grpc-all - - - io.netty - * - - - - - - io.perfmark - perfmark-api - runtime - - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - - - - generate-sources - - compile - compile-custom - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 3.1.0-SNAPSHOT + + pulsar-functions-proto + Pulsar Functions :: Proto + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + javax.annotation + javax.annotation-api + + + io.grpc + grpc-all + + + io.netty + * + + + + + io.perfmark + perfmark-api + runtime + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + true + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + + + + generate-sources + + compile + compile-custom + + + + + + diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 55e4009db077e..eb8ae5646ada7 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT .. - - pulsar-functions-runtime-all Pulsar Functions :: Runtime All - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-client-api ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.protobuf @@ -95,30 +85,24 @@ gson ${gson.version} - - org.slf4j slf4j-api - org.apache.logging.log4j log4j-slf4j-impl - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index 190d9be92940b..84e6ce44f51e4 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - info - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + info + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index f86d03e41793f..fe478be5b8667 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-kubernetes-instance - 30 - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - Console - - - - info - - Console - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-kubernetes-instance + 30 + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + Console + + + + info + + Console + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 728d3f80e1965..f8d039a3073ac 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-runtime Pulsar Functions :: Runtime - - ${project.groupId} pulsar-functions-instance ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - com.beust jcommander - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-databind - io.kubernetes client-java @@ -84,9 +75,7 @@ pulsar-broker-common ${project.version} - - @@ -106,7 +95,6 @@ - org.apache.maven.plugins @@ -154,5 +142,4 @@ - diff --git a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml index b4a4e5926de3f..976abe7c54bb2 100644 --- a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 93069bd4a3b64..db7c2d2712e89 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-secrets Pulsar Functions :: Secrets - io.kubernetes @@ -50,7 +48,6 @@ - ${project.groupId} pulsar-functions-proto @@ -76,7 +73,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -109,5 +105,4 @@ - diff --git a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml index 0cab8ae6a418a..7d4464ae1c08d 100644 --- a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT - pulsar-functions-utils Pulsar Functions :: Utils - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-proto ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-config-validation ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.core jackson-databind - com.google.code.gson gson - net.jodah typetools - - ${project.groupId} - pulsar-client-original - ${project.version} + ${project.groupId} + pulsar-client-original + ${project.version} - ${project.groupId} pulsar-package-core ${project.version} - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - @@ -126,7 +110,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -142,5 +125,4 @@ - diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index c6db572f57d7e..cbfd2bf37aac0 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.1.0-SNAPSHOT .. - pulsar-functions-worker Pulsar Functions :: Worker - - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-functions-runtime ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-multipart - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.apache.distributedlog distributedlog-core @@ -118,17 +100,14 @@ - org.apache.bookkeeper bookkeeper-server - commons-io commons-io - ${project.groupId} pulsar-docs-tools @@ -140,7 +119,6 @@ - io.swagger swagger-core @@ -156,14 +134,11 @@ - io.prometheus simpleclient_jetty - - ${project.groupId} pulsar-io-cassandra @@ -171,7 +146,6 @@ pom test - ${project.groupId} pulsar-io-twitter @@ -179,7 +153,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples @@ -187,7 +160,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples-builtin @@ -195,9 +167,7 @@ pom test - - @@ -217,7 +187,6 @@ - maven-dependency-plugin @@ -270,7 +239,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -286,7 +254,6 @@ - org.apache.maven.plugins @@ -334,5 +301,4 @@ - diff --git a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 7f604add58819..cf648b8dfb6ea 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-aerospike Pulsar IO :: Aerospike - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.aerospike aerospike-client-bc @@ -63,9 +57,7 @@ org.bouncycastle bcpkix-jdk18on - - @@ -91,6 +83,4 @@ - - diff --git a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/alluxio/pom.xml b/pulsar-io/alluxio/pom.xml index 878194d8bd2ba..6101573b33f3d 100644 --- a/pulsar-io/alluxio/pom.xml +++ b/pulsar-io/alluxio/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.1.0-SNAPSHOT - - - - 2.7.3 - 4.1.11 - 1.37.0 - - - pulsar-io-alluxio - Pulsar IO :: Alluxio - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - test - - - - org.alluxio - alluxio-core-client-fs - ${alluxio.version} - - - grpc-netty - io.grpc - - - - - - org.alluxio - alluxio-minicluster - ${alluxio.version} - test - - - org.glassfish - javax.el - - - grpc-netty - io.grpc - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - - - - - io.grpc - grpc-netty - ${grpc.version} - - - - io.dropwizard.metrics - metrics-jvm - ${metrics.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + pulsar-io + io.streamnative + 3.1.0-SNAPSHOT + + + 2.7.3 + 4.1.11 + 1.37.0 + + pulsar-io-alluxio + Pulsar IO :: Alluxio + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + test + + + org.alluxio + alluxio-core-client-fs + ${alluxio.version} + + + grpc-netty + io.grpc + + + + + org.alluxio + alluxio-minicluster + ${alluxio.version} + test + + + org.glassfish + javax.el + + + grpc-netty + io.grpc + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.google.guava + guava + + + + io.grpc + grpc-netty + ${grpc.version} + + + io.dropwizard.metrics + metrics-jvm + ${metrics.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 74e74f6e25851..1bb9efee198ae 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-aws Pulsar IO :: IO AWS - ${project.groupId} pulsar-io-core ${project.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-sts - software.amazon.awssdk sts 2.10.56 - - + - diff --git a/pulsar-io/aws/src/main/resources/findbugsExclude.xml b/pulsar-io/aws/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aws/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aws/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 1a6bffd531ffb..3ab06875d0c17 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - - pulsar-io-batch-data-generator - Pulsar IO :: Batch Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-io-batch-discovery-triggerers - ${project.version} - - - - org.springframework - spring-context - ${spring.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT + + pulsar-io-batch-data-generator + Pulsar IO :: Batch Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-io-batch-discovery-triggerers + ${project.version} + + + org.springframework + spring-context + ${spring.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-functions-local-runner-original + ${project.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml index 0b1652ef62881..1d0d64fba7e67 100644 --- a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-batch-discovery-triggerers Pulsar IO :: Batch Discovery Triggerers - ${project.groupId} pulsar-io-core ${project.version} - com.cronutils cron-utils ${cron-utils.version} - org.springframework spring-context ${spring.version} - - @@ -73,5 +66,4 @@ - diff --git a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - 4.0.0 - - pulsar-io-canal - Pulsar IO :: Canal - - - 1.1.5 - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.alibaba - fastjson - 1.2.83 - - - - org.springframework - spring-core - ${spring.version} - - - org.springframework - spring-aop - ${spring.version} - - - org.springframework - spring-context - ${spring.version} - - - org.springframework - spring-jdbc - ${spring.version} - - - org.springframework - spring-orm - ${spring.version} - - - - com.alibaba.otter - canal.protocol - ${canal.version} - - - ch.qos.logback - * - - - - - com.alibaba.otter - canal.client - ${canal.version} - - - ch.qos.logback - * - - - org.springframework - * - - - - - - org.apache.logging.log4j - log4j-core - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - \ No newline at end of file + + + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT + + 4.0.0 + pulsar-io-canal + Pulsar IO :: Canal + + 1.1.5 + + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.alibaba + fastjson + 1.2.83 + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-jdbc + ${spring.version} + + + org.springframework + spring-orm + ${spring.version} + + + com.alibaba.otter + canal.protocol + ${canal.version} + + + ch.qos.logback + * + + + + + com.alibaba.otter + canal.client + ${canal.version} + + + ch.qos.logback + * + + + org.springframework + * + + + + + org.apache.logging.log4j + log4j-core + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 3859bd1029e86..8e016fe895860 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-cassandra Pulsar IO :: Cassandra - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.datastax.cassandra cassandra-driver-core - - diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 14a134bc42858..7da845df776ca 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - - pulsar-io-common - Pulsar IO :: IO Common - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT + + pulsar-io-common + Pulsar IO :: IO Common + + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/common/src/main/resources/findbugsExclude.xml b/pulsar-io/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-core Pulsar IO :: IO - ${project.groupId} @@ -37,7 +35,6 @@ ${project.version} - @@ -59,5 +56,4 @@ - diff --git a/pulsar-io/core/src/main/resources/findbugsExclude.xml b/pulsar-io/core/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - - pulsar-io-data-generator - Pulsar IO :: Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-config-validation - ${project.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT + + pulsar-io-data-generator + Pulsar IO :: Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-config-validation + ${project.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 2e5a6f5c24da9..715e654d5e370 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.1.0-SNAPSHOT - pulsar-io-debezium-core Pulsar IO :: Debezium :: Core - - ${project.groupId} pulsar-io-core ${project.version} provided - io.debezium debezium-core ${debezium.version} - org.apache.commons commons-lang3 - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-kafka-connect-adaptor ${project.version} - org.apache.kafka connect-runtime @@ -73,21 +65,18 @@ - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -95,21 +84,17 @@ test test-jar - io.debezium debezium-connector-mysql ${debezium.version} test - ${project.groupId} pulsar-client-original ${project.version} test - - diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index ad57e0de02b77..f187d71bec204 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io-debezium - 3.1.0-SNAPSHOT - - - pulsar-io-debezium-mongodb - Pulsar IO :: Debezium :: mongodb - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - provided - - - - ${project.groupId} - pulsar-io-debezium-core - ${project.version} - - - - io.debezium - debezium-connector-mongodb - ${debezium.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - + + 4.0.0 + + io.streamnative + pulsar-io-debezium + 3.1.0-SNAPSHOT + + pulsar-io-debezium-mongodb + Pulsar IO :: Debezium :: mongodb + + + ${project.groupId} + pulsar-io-core + ${project.version} + provided + + + ${project.groupId} + pulsar-io-debezium-core + ${project.version} + + + io.debezium + debezium-connector-mongodb + ${debezium.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 4db8477f185bc..fe10f2b269e2c 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.1.0-SNAPSHOT - pulsar-io-debezium-mssql Pulsar IO :: Debezium :: Microsoft SQL - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-sqlserver ${debezium.version} - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 8c443f2f6b6df..691b2cc537e50 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.1.0-SNAPSHOT - pulsar-io-debezium-mysql Pulsar IO :: Debezium :: mysql - ${project.groupId} @@ -37,21 +35,17 @@ ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-mysql ${debezium.version} - - @@ -61,8 +55,6 @@ - - @@ -71,5 +63,4 @@ - diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 9d6bffd38f2d2..ebd8ff0585559 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.1.0-SNAPSHOT - pulsar-io-debezium-oracle Pulsar IO :: Debezium :: oracle - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-oracle ${debezium.version} - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index ba64915466710..e64835abcd848 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-debezium Pulsar IO :: Debezium - @@ -70,7 +68,6 @@ - core mysql @@ -79,5 +76,4 @@ oracle mssql - diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 86fba2d3d1f34..a2e271e8ba1a3 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.1.0-SNAPSHOT - pulsar-io-debezium-postgres Pulsar IO :: Debezium :: postgres - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-postgres ${debezium.version} - org.postgresql postgresql ${debezium.postgresql.version} runtime - - @@ -68,5 +60,4 @@ - diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 0c772104dbbfc..e200ba1f116b4 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-docs Pulsar IO :: Docs - - ${project.groupId} pulsar-io-core @@ -45,7 +42,6 @@ com.beust jcommander - ${project.groupId} @@ -218,7 +214,6 @@ ${project.version} - @@ -267,5 +262,4 @@ - diff --git a/pulsar-io/docs/src/main/resources/findbugsExclude.xml b/pulsar-io/docs/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/docs/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/docs/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-dynamodb Pulsar IO :: DynamoDB - - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-instance ${project.version} provided - ${project.groupId} pulsar-io-aws ${project.version} - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - com.amazonaws @@ -90,9 +78,7 @@ 1.5.1 - - @@ -101,5 +87,4 @@ - diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 24c0bdc052841..0a757a8ab018e 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - - pulsar-io-flume - Pulsar IO :: Flume - - - 1.8.2 - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - org.apache.flume - flume-ng-node - 1.9.0 - pom - - - org.apache.avro - avro-ipc - - - avro - org.apache.avro - - - - - org.apache.avro - avro - ${avro.version} - - - commons-lang - commons-lang - 2.6 - - - org.apache.avro - avro-ipc - ${avro.version} - - - org.mortbay.jetty - servlet-api - - - io.netty - netty - - - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - curator-test - ${curator.version} - test - - - - io.dropwizard.metrics - metrics-core - test - - - - org.xerial.snappy - snappy-java - test - - - - com.google.guava - guava - ${guava.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + com.google.guava + guava + ${guava.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index ae2845a791dcd..22e6fec3cb115 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.1.0-SNAPSHOT - - pulsar-io-hbase - Pulsar IO :: Hbase - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - - - - org.apache.hbase - hbase-client - ${hbase.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hbase - hbase-common - ${hbase.version} - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml index c2b520a765643..460cbd497c15f 100644 --- a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml +++ b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml @@ -1,3 +1,4 @@ + - - hbase.cluster.distributed - true - - - hbase.rootdir - hdfs://localhost:8020/hbase - - - hbase.zookeeper.quorum - localhost - - - hbase.zookeeper.property.clientPort - 2181 - - - zookeeper.znode.parent - /hbase - + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://localhost:8020/hbase + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + + zookeeper.znode.parent + /hbase + diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 7b2f11c7b7c21..46759ef7f4282 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -1,3 +1,4 @@ + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - - \ No newline at end of file + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + + diff --git a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml index 980349c4d8d5e..93756f7f28461 100644 --- a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 30f7368102df4..6935bace2aa64 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index 47e48648e570e..6090fdd5dd623 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - - pulsar-io-http - Pulsar IO :: HTTP - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - - org.apache.avro - avro - ${avro.version} - - - - org.apache.pulsar - pulsar-client-original - ${project.version} - test - - - - com.github.tomakehurst - wiremock-jre8 - ${wiremock.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT + + pulsar-io-http + Pulsar IO :: HTTP + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + org.apache.avro + avro + ${avro.version} + + + io.streamnative + pulsar-client-original + ${project.version} + test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index cad7956ccfcce..f865518b9f97d 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.1.0-SNAPSHOT - - - pulsar-io-influxdb - Pulsar IO :: InfluxDB - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.influxdb - influxdb-client-java - 4.0.0 - - - - org.influxdb - influxdb-java - 2.22 - - - com.squareup.okhttp3 - * - - - - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.1.0-SNAPSHOT + + pulsar-io-influxdb + Pulsar IO :: InfluxDB + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + com.influxdb + influxdb-client-java + 4.0.0 + + + org.influxdb + influxdb-java + 2.22 + + + com.squareup.okhttp3 + * + + + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 14dc030a753e4..065fe08835dc8 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-clickhouse Pulsar IO :: Jdbc :: ClickHouse - ${project.groupId} @@ -45,13 +42,12 @@ all - * - * + * + * - diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 5c2e97c7fdc29..6e0ef882f5eab 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-core Pulsar IO :: Jdbc :: Core - ${project.groupId} pulsar-io-core ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.guava guava - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.protobuf protobuf-java provided - - - \ No newline at end of file + diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 943406a8b900f..b54b3a25d8a66 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-mariadb Pulsar IO :: Jdbc :: Mariadb - ${project.groupId} @@ -52,4 +49,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index 92376583d25ba..4623729710323 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-openmldb Pulsar IO :: Jdbc :: OpenMLDB - ${project.groupId} @@ -60,7 +57,6 @@ runtime - diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 5a82e163c554b..88e640621ea85 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom @@ -31,12 +31,10 @@ openmldb - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-jdbc Pulsar IO :: Jdbc - diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index e7177cb2e19aa..648b7f2df008b 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-postgres Pulsar IO :: Jdbc :: Postgres - ${project.groupId} @@ -44,7 +41,6 @@ runtime - diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 5cf8833e6684a..7a8e932deb0da 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite Pulsar IO :: Jdbc :: Sqlite - ${project.groupId} @@ -60,7 +58,6 @@ test - @@ -69,4 +66,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index b3d547d333127..6255482116b8f 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-kafka-connect-adaptor-nar Pulsar IO :: Kafka Connect Adaptor NAR - ${project.groupId} @@ -37,8 +35,6 @@ ${project.version} - - @@ -48,7 +44,30 @@ pulsar-io-kafka-connect-adaptor-${project.version} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 71a3cee11dcd0..a70450d12cf3e 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-kafka-connect-adaptor Pulsar IO :: Kafka Connect Adaptor - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-common ${project.version} compile - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.kafka connect-runtime @@ -75,19 +68,16 @@ - org.apache.kafka connect-json ${kafka-client.version} - org.apache.kafka connect-api ${kafka-client.version} - ${project.groupId} @@ -100,45 +90,38 @@ - io.netty netty-buffer - org.apache.commons commons-lang3 - io.confluent kafka-connect-avro-converter ${confluent.version} - ${project.groupId} pulsar-broker ${project.version} test - org.apache.kafka connect-file ${kafka-client.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -146,38 +129,37 @@ test test-jar - org.apache.avro avro ${avro.version} provided - com.google.protobuf protobuf-java provided - org.asynchttpclient async-http-client test - org.bouncycastle bc-fips ${bouncycastle.bc-fips.version} test - com.typesafe.netty netty-reactive-streams test + + com.google.protobuf + protobuf-java + 3.5.1 + - diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 28971ac282af1..634eda900a4e1 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-kafka Pulsar IO :: Kafka - @@ -44,79 +42,64 @@ - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - org.apache.kafka kafka-clients ${kafka-client.version} - io.confluent kafka-schema-registry-client ${confluent.version} - io.confluent kafka-avro-serializer ${confluent.version} - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.hamcrest hamcrest test - org.awaitility awaitility test - - diff --git a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml index 3fc8f67c8640e..a48959506a69a 100644 --- a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-kinesis Pulsar IO :: Kinesis - 2.2.8 0.14.0 @@ -37,116 +35,96 @@ 1.9.0 2.3.0 - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-io-aws ${project.version} - - org.apache.pulsar + io.streamnative pulsar-functions-instance ${project.version} test - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - org.apache.avro avro ${avro.version} - com.github.wnameless.json json-flattener ${json-flattener.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - software.amazon.kinesis amazon-kinesis-client ${amazon-kinesis-client.version} - com.amazonaws amazon-kinesis-producer ${amazon-kinesis-producer.version} - - + com.google.flatbuffers flatbuffers-java ${flatbuffers-java.version} - javax.xml.bind jaxb-api ${jaxb-api.version} - org.testcontainers localstack test - org.awaitility awaitility test - - @@ -155,5 +133,4 @@ - diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 214e76bad7927..ea067be6e73c3 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT - pulsar-io-mongo Pulsar IO :: MongoDB - 4.1.2 - ${project.parent.groupId} @@ -64,7 +59,6 @@ guava - diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index dcbc025510b67..56834efb878a5 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.1.0-SNAPSHOT - - - pulsar-io-netty - Pulsar IO :: Netty - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - io.netty - netty-handler - - - - io.netty - netty-codec-http - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.1.0-SNAPSHOT + + pulsar-io-netty + Pulsar IO :: Netty + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + io.netty + netty-handler + + + io.netty + netty-codec-http + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/netty/src/main/resources/findbugsExclude.xml b/pulsar-io/netty/src/main/resources/findbugsExclude.xml index 3a52062fc9233..6b8906a775bd3 100644 --- a/pulsar-io/netty/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/netty/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index f6bed0c7a74b4..a79d424ec4b62 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-nsq Pulsar IO :: NSQ - ${project.groupId} pulsar-io-core ${project.version} - com.sproutsocial nsq-j ${nsq-client.version} - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - ${project.groupId} pulsar-io-common ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - - - - org.apache.nifi - nifi-nar-maven-plugin - - + + + org.apache.nifi + nifi-nar-maven-plugin + + diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index c1a58d059cd48..8b005ef65df78 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-io Pulsar IO :: Parent - main @@ -77,7 +75,6 @@ alluxio - pulsar-io-tests @@ -110,7 +107,6 @@ alluxio - pulsar-io-elastic-tests @@ -119,7 +115,6 @@ elastic-search - pulsar-io-kafka-connect-tests @@ -131,7 +126,6 @@ debezium - core-modules @@ -145,7 +139,6 @@ - @@ -163,5 +156,4 @@ - diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 3cdcb681b3fa8..3eae092f070f2 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-rabbitmq Pulsar IO :: RabbitMQ - - ${project.groupId} pulsar-io-core @@ -53,28 +50,23 @@ pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - com.rabbitmq amqp-client ${rabbitmq-client.version} - org.apache.qpid qpid-broker @@ -86,9 +78,7 @@ awaitility test - - @@ -97,5 +87,4 @@ - diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 351ae205e667f..b3cb6ed7152ac 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.1.0-SNAPSHOT - - - pulsar-io-redis - Pulsar IO :: Redis - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - io.lettuce - lettuce-core - 5.0.2.RELEASE - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - com.github.kstyrc - embedded-redis - 0.6 - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.1.0-SNAPSHOT + + pulsar-io-redis + Pulsar IO :: Redis + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + io.lettuce + lettuce-core + 5.0.2.RELEASE + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + com.github.kstyrc + embedded-redis + 0.6 + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 741c6d135c509..56caf141bdf0b 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.1.0-SNAPSHOT - - - - 8.11.1 - - - pulsar-io-solr - Pulsar IO :: Solr - - - - ${project.parent.groupId} - pulsar-io-core - ${project.parent.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - org.apache.solr - solr-solrj - ${solr.version} - - - org.apache.solr - solr-core - ${solr.version} - test - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.1.0-SNAPSHOT + + + 8.11.1 + + pulsar-io-solr + Pulsar IO :: Solr + + + ${project.parent.groupId} + pulsar-io-core + ${project.parent.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.apache.solr + solr-core + ${solr.version} + test + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + + diff --git a/pulsar-io/solr/src/main/resources/findbugsExclude.xml b/pulsar-io/solr/src/main/resources/findbugsExclude.xml index f49ff86c90caf..47c3d73af5f06 100644 --- a/pulsar-io/solr/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/solr/src/main/resources/findbugsExclude.xml @@ -1,22 +1,23 @@ - - - \ No newline at end of file + + + + diff --git a/pulsar-io/solr/src/test/resources/solr.xml b/pulsar-io/solr/src/test/resources/solr.xml index 495a71f72f3f2..9ce4d37f4beef 100644 --- a/pulsar-io/solr/src/test/resources/solr.xml +++ b/pulsar-io/solr/src/test/resources/solr.xml @@ -1,3 +1,4 @@ + - - ${host:} - ${jetty.port:8983} - ${hostContext:solr} - ${genericCoreNodeNames:true} - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - ${socketTimeout:600000} - ${connTimeout:60000} - - \ No newline at end of file + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + ${genericCoreNodeNames:true} + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + ${socketTimeout:600000} + ${connTimeout:60000} + + diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 03b9394c44109..41374d0fee261 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.1.0-SNAPSHOT - pulsar-io-twitter Pulsar IO :: Twitter - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.twitter hbc-core ${hbc-core.version} - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - - ${project.groupId} - pulsar-io-common - ${project.version} + ${project.groupId} + pulsar-io-common + ${project.version} - - @@ -80,5 +69,4 @@ - diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index a494fa1c35f8f..22a73d2b0617a 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-metadata Pulsar Metadata - org.apache.pulsar + io.streamnative pulsar-common ${project.version} - org.apache.zookeeper zookeeper @@ -56,56 +53,45 @@ - io.dropwizard.metrics metrics-core test - org.xerial.snappy snappy-java test - org.awaitility awaitility test - org.apache.bookkeeper bookkeeper-server - io.etcd jetcd-core - - io.etcd jetcd-test test - com.github.ben-manes.caffeine caffeine - io.prometheus simpleclient - - @@ -125,7 +111,6 @@ - org.apache.maven.plugins maven-jar-plugin diff --git a/pulsar-metadata/src/main/resources/findbugsExclude.xml b/pulsar-metadata/src/main/resources/findbugsExclude.xml index 21578b9f8cb5c..d6eb8623eb73e 100644 --- a/pulsar-metadata/src/main/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-metadata/src/test/resources/findbugsExclude.xml b/pulsar-metadata/src/test/resources/findbugsExclude.xml index e40b2a0fd9a12..bf075c39791cf 100644 --- a/pulsar-metadata/src/test/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index eda8edbf93048..eea43b6aa4235 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 3.1.0-SNAPSHOT - - 4.0.0 - - pulsar-package-bookkeeper-storage - Apache Pulsar :: Package Management :: BookKeeper Storage - - - - ${project.groupId} - pulsar-package-core - ${project.version} - - - - org.apache.distributedlog - distributedlog-core - - - net.jpountz.lz4 - lz4 - - - - - - org.apache.zookeeper - zookeeper - tests - test - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - io.netty - netty-tcnative - - - - - - - io.dropwizard.metrics - metrics-core - test - - - - - org.xerial.snappy - snappy-java - test - - - - ${project.groupId} - managed-ledger - ${project.version} - test - - - - ${project.groupId} - testmocks - ${project.version} - - - + + + pulsar-package-management + io.streamnative + 3.1.0-SNAPSHOT + + 4.0.0 + pulsar-package-bookkeeper-storage + Apache Pulsar :: Package Management :: BookKeeper Storage + + + ${project.groupId} + pulsar-package-core + ${project.version} + + + org.apache.distributedlog + distributedlog-core + + + net.jpountz.lz4 + lz4 + + + + + org.apache.zookeeper + zookeeper + tests + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + ${project.groupId} + managed-ledger + ${project.version} + test + + + ${project.groupId} + testmocks + ${project.version} + + diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 605c4c13e61fa..242bb67d156d7 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 3.1.0-SNAPSHOT - - 4.0.0 - - pulsar-package-core - Apache Pulsar :: Package Management :: Core - - - - ${project.groupId} - pulsar-client-admin-api - ${project.parent.version} - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 17 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - + + + pulsar-package-management + io.streamnative + 3.1.0-SNAPSHOT + + 4.0.0 + pulsar-package-core + Apache Pulsar :: Package Management :: Core + + + ${project.groupId} + pulsar-client-admin-api + ${project.parent.version} + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 17 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + diff --git a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml index 3a2e998dce984..bd8d2c4f77efa 100644 --- a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - pulsar-package-management - org.apache.pulsar - 3.1.0-SNAPSHOT - - 4.0.0 - - pulsar-package-filesystem-storage - Apache Pulsar :: Package Management :: Filesystem Storage - - - - ${project.groupId} - pulsar-package-core - ${project.parent.version} - - - - com.google.guava - guava - - - - ${project.groupId} - testmocks - ${project.parent.version} - test - - - + + + pulsar-package-management + io.streamnative + 3.1.0-SNAPSHOT + + 4.0.0 + pulsar-package-filesystem-storage + Apache Pulsar :: Package Management :: Filesystem Storage + + + ${project.groupId} + pulsar-package-core + ${project.parent.version} + + + com.google.guava + guava + + + ${project.groupId} + testmocks + ${project.parent.version} + test + + diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 3a61f659d3109..aedc954344f49 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar - org.apache.pulsar - 3.1.0-SNAPSHOT - .. - - 4.0.0 - - pulsar-package-management - pom - Apache Pulsar :: Package Management - - core - bookkeeper-storage - filesystem-storage - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - spotbugs - verify - - check - - - - - - + + + pulsar + io.streamnative + 3.1.0-SNAPSHOT + .. + + 4.0.0 + pulsar-package-management + pom + Apache Pulsar :: Package Management + + core + bookkeeper-storage + filesystem-storage + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index de1eb9a16c05b..ae6f93e7c64f8 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-proxy Pulsar Proxy - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-docs-tools ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - org.apache.commons commons-lang3 - io.swagger swagger-annotations provided - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.eclipse.jetty jetty-proxy - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-json-jackson - com.google.guava guava - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api - com.sun.activation javax.activation - io.prometheus simpleclient - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_servlet - io.prometheus simpleclient_jetty - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - org.awaitility awaitility test - com.beust jcommander - org.apache.logging.log4j log4j-core - com.github.seancfoley ipaddress @@ -295,5 +264,4 @@ - diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index d027e1a56c578..8340226ae59a2 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - pom - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - - - pulsar-sql - Pulsar SQL :: Parent - - - - 3.14.9 - - 1.17.2 - 213 - - - - - - - com.squareup.okhttp3 - okhttp - ${okhttp3.version} - - - com.squareup.okhttp3 - okhttp-urlconnection - ${okhttp3.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp3.version} - - - com.squareup.okio - okio - ${okio.version} - - - - - org.jline - jline-reader - ${jline3.version} - - - org.jline - jline-terminal - ${jline3.version} - - - org.jline - jline-terminal-jna - ${jline3.version} - - - - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-jdk14 - ${slf4j.version} - - - - io.airlift - bom - ${airlift.version} - pom - import - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - main - - - disableSqlMainProfile - - !true - - - - presto-pulsar - presto-pulsar-plugin - presto-distribution - - - - pulsar-sql-tests - - presto-pulsar - presto-pulsar-plugin - presto-distribution - - - + 3.14.9 + + 1.17.2 + 213 + + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.okhttp3 + okhttp-urlconnection + ${okhttp3.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + com.squareup.okio + okio + ${okio.version} + + + + org.jline + jline-reader + ${jline3.version} + + + org.jline + jline-terminal + ${jline3.version} + + + org.jline + jline-terminal-jna + ${jline3.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-jdk14 + ${slf4j.version} + + + io.airlift + bom + ${airlift.version} + pom + import + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + + + main + + + disableSqlMainProfile + + !true + + + + presto-pulsar + presto-pulsar-plugin + presto-distribution + + + + pulsar-sql-tests + + presto-pulsar + presto-pulsar-plugin + presto-distribution + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 1ac764a3c64d9..dc1e24ee490d1 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-sql 3.1.0-SNAPSHOT - pulsar-presto-distribution Pulsar SQL :: Pulsar Presto Distribution - false 2.34 @@ -42,7 +39,6 @@ 2.5.1 4.0.1 - org.glassfish.jersey.core @@ -74,7 +70,6 @@ jersey-client ${jersey.version} - io.trino trino-server-main @@ -103,13 +98,11 @@ - io.trino trino-cli ${trino.version} - io.airlift launcher @@ -117,7 +110,6 @@ tar.gz bin - io.airlift launcher @@ -125,13 +117,11 @@ tar.gz properties - org.objenesis objenesis ${objenesis.version} - com.twitter.common objectsize @@ -143,56 +133,44 @@ - - com.fasterxml.jackson.core jackson-core - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.datatype jackson-datatype-joda - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.datatype jackson-datatype-guava - com.fasterxml.jackson.datatype jackson-datatype-jdk8 - com.fasterxml.jackson.datatype jackson-datatype-jsr310 - com.fasterxml.jackson.dataformat jackson-dataformat-smile - - @@ -205,7 +183,6 @@ netty 3.10.6.Final - org.apache.maven maven-core @@ -295,7 +272,6 @@ - @@ -305,7 +281,6 @@ ${skipBuildDistribution} - org.apache.maven.plugins maven-assembly-plugin @@ -329,7 +304,6 @@ - com.mycila license-maven-plugin @@ -354,7 +328,6 @@ - skipBuildDistributionDisabled diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml index 96c0421c71515..8439d60ba339e 100644 --- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml +++ b/pulsar-sql/presto-distribution/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + - - bin - - tar.gz - dir - - false - - - ${basedir}/LICENSE - LICENSE - . - 644 - - - ${basedir}/src/main/resources/launcher.properties - launcher.properties - bin/ - 644 - - - - - ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ - plugin/ - - - ${basedir}/src/main/resources/conf/ - conf/ - - - - - lib/ - true - runtime - - io.airlift:launcher:tar.gz:bin:${airlift.version} - io.airlift:launcher:tar.gz:properties:${airlift.version} - *:tar.gz - - - - - - io.airlift:launcher:tar.gz:bin:${airlift.version} - - true - 755 - - - - - io.airlift:launcher:tar.gz:properties:${airlift.version} - - true - - - \ No newline at end of file + + bin + + tar.gz + dir + + false + + + ${basedir}/LICENSE + LICENSE + . + 644 + + + ${basedir}/src/main/resources/launcher.properties + launcher.properties + bin/ + 644 + + + + + ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ + plugin/ + + + ${basedir}/src/main/resources/conf/ + conf/ + + + + + lib/ + true + runtime + + io.airlift:launcher:tar.gz:bin:${airlift.version} + io.airlift:launcher:tar.gz:properties:${airlift.version} + *:tar.gz + + + + + + io.airlift:launcher:tar.gz:bin:${airlift.version} + + true + 755 + + + + + io.airlift:launcher:tar.gz:properties:${airlift.version} + + true + + + diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 5e88a24bba530..3b8616ed318cd 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.1.0-SNAPSHOT - - - pulsar-presto-connector - Pulsar SQL :: Pulsar Presto Connector - - - - - ${project.groupId} - pulsar-presto-connector-original - ${project.version} - - - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.3.0 - - false - true - posix - - src/assembly/assembly.xml - - - - - package - package - - single - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-sql + 3.1.0-SNAPSHOT + + pulsar-presto-connector + Pulsar SQL :: Pulsar Presto Connector + + + ${project.groupId} + pulsar-presto-connector-original + ${project.version} + + + ${project.groupId} + bouncy-castle-bc + ${project.version} + pkg + true + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + false + true + posix + + src/assembly/assembly.xml + + + + + package + package + + single + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml index 6650abfda3fc3..649baf7318003 100644 --- a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml +++ b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + - - bin - - tar.gz - dir - - - - / - false - runtime - - jakarta.ws.rs:jakarta.ws.rs-api - - - - \ No newline at end of file + + bin + + tar.gz + dir + + + + / + false + runtime + + jakarta.ws.rs:jakarta.ws.rs-api + + + + diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 3ff073f62e459..235445ade9db6 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.1.0-SNAPSHOT - - - pulsar-presto-connector-original - Pulsar SQL - Pulsar Presto Connector - Pulsar SQL :: Pulsar Presto Connector Packaging - - - 2.1.2 - 1.8.4 - - - - - io.airlift - bootstrap - - - org.apache.logging.log4j - log4j-to-slf4j - - - - - io.airlift - json - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - org.jctools - jctools-core - ${jctools.version} - - - - com.dslplatform - dsl-json - ${dslJson.verson} - - - - io.trino - trino-plugin-toolkit - ${trino.version} - - - - - io.trino - trino-spi - ${trino.version} - provided - - - - joda-time - joda-time - ${joda.version} - - - - io.trino - trino-record-decoder - ${trino.version} - - - - javax.annotation - javax.annotation-api - ${javax.annotation-api.version} - - - - io.jsonwebtoken - jjwt-impl - ${jsonwebtoken.version} - test - - - - io.trino - trino-main - ${trino.version} - test - - - - io.trino - trino-testing - ${trino.version} - test - - - - org.apache.pulsar + + 4.0.0 + + io.streamnative + pulsar-sql + 3.1.0-SNAPSHOT + + pulsar-presto-connector-original + Pulsar SQL - Pulsar Presto Connector + Pulsar SQL :: Pulsar Presto Connector Packaging + + 2.1.2 + 1.8.4 + + + + io.airlift + bootstrap + + + org.apache.logging.log4j + log4j-to-slf4j + + + + + io.airlift + json + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.jctools + jctools-core + ${jctools.version} + + + com.dslplatform + dsl-json + ${dslJson.verson} + + + io.trino + trino-plugin-toolkit + ${trino.version} + + + + io.trino + trino-spi + ${trino.version} + provided + + + joda-time + joda-time + ${joda.version} + + + io.trino + trino-record-decoder + ${trino.version} + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + io.jsonwebtoken + jjwt-impl + ${jsonwebtoken.version} + test + + + io.trino + trino-main + ${trino.version} + test + + + io.trino + trino-testing + ${trino.version} + test + + + io.streamnative + pulsar-broker + ${project.version} + test + + + io.streamnative + testmocks + ${project.version} + test + + + org.eclipse.jetty + jetty-http + ${jetty.version} + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + true + true + + + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original + io.streamnative:managed-ledger + io.streamnative:pulsar-metadata + org.glassfish.jersey*:* + javax.ws.rs:* + javax.annotation:* + org.glassfish.hk2*:* + org.eclipse.jetty:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + org.glassfish + org.apache.pulsar.shade.org.glassfish + + + javax.ws + org.apache.pulsar.shade.javax.ws + + + javax.annotation + org.apache.pulsar.shade.javax.annotation + + + jersey + org.apache.pulsar.shade.jersey + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + + + + + + + + + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} pulsar-broker ${project.version} + test-jar test - - - org.apache.pulsar - testmocks - ${project.version} - test - - - - org.eclipse.jetty - jetty-http - ${jetty.version} - test - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - true - true - - - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original - org.apache.pulsar:managed-ledger - org.apache.pulsar:pulsar-metadata - - org.glassfish.jersey*:* - javax.ws.rs:* - javax.annotation:* - org.glassfish.hk2*:* - - org.eclipse.jetty:* - - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - org.glassfish - org.apache.pulsar.shade.org.glassfish - - - javax.ws - org.apache.pulsar.shade.javax.ws - - - javax.annotation - org.apache.pulsar.shade.javax.annotation - - - jersey - org.apache.pulsar.shade.jersey - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - - - - - - - - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - + + + diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index ce03dd444e101..dce6cf0c19b63 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - .. - - - pulsar-testclient - Pulsar Test Client - Pulsar Test Client - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.zookeeper - zookeeper - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - io.netty - netty-tcnative - - - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - ${project.groupId} - pulsar-client-messagecrypto-bc - ${project.version} - true - - - - ${project.groupId} - pulsar-broker - ${project.version} - - - - commons-configuration - commons-configuration - - - - com.beust - jcommander - compile - - - - org.hdrhistogram - HdrHistogram - - - - com.fasterxml.jackson.core - jackson-databind - - - - org.awaitility - awaitility - test - - - - com.github.tomakehurst - wiremock-jre8 - ${wiremock.version} - test - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 3.1.0-SNAPSHOT + .. + + pulsar-testclient + Pulsar Test Client + Pulsar Test Client + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.zookeeper + zookeeper + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + ${project.groupId} + pulsar-client-messagecrypto-bc + ${project.version} + true + + + ${project.groupId} + pulsar-broker + ${project.version} + + + commons-configuration + commons-configuration + + + com.beust + jcommander + compile + + + org.hdrhistogram + HdrHistogram + + + com.fasterxml.jackson.core + jackson-databind + + + org.awaitility + awaitility + test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} + pulsar-broker + ${project.version} + test-jar + test + + + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index b2594bf26ade5..6ab440cf883fe 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 3.1.0-SNAPSHOT - - - pulsar-transaction-common - Pulsar Transaction :: Common - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 3.1.0-SNAPSHOT + + pulsar-transaction-common + Pulsar Transaction :: Common + - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 3.1.0-SNAPSHOT - - - pulsar-transaction-coordinator - Pulsar Transaction :: Coordinator - - - - - ${project.groupId} - pulsar-common - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.awaitility - awaitility - test - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - proto_path - ${project.parent.parent.basedir} - - - proto_search_strategy - 2 - - - - - - - - com.github.splunk.lightproto - lightproto-maven-plugin - ${lightproto-maven-plugin.version} - - - - generate - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 3.1.0-SNAPSHOT + + pulsar-transaction-coordinator + Pulsar Transaction :: Coordinator + + + ${project.groupId} + pulsar-common + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.awaitility + awaitility + test + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.codehaus.mojo + properties-maven-plugin + + + initialize + + set-system-properties + + + + + proto_path + ${project.parent.parent.basedir} + + + proto_search_strategy + 2 + + + + + + + + com.github.splunk.lightproto + lightproto-maven-plugin + ${lightproto-maven-plugin.version} + + + + generate + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml index a81fce11f4d21..f1cad02b28a88 100644 --- a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - pulsar-transaction-parent Pulsar Transaction :: Parent - common coordinator - - - com.github.spotbugs spotbugs-maven-plugin @@ -68,7 +63,5 @@ - - diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index bf8f38f2da1af..bb5e5707af6b8 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - pulsar-websocket Pulsar WebSocket - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} managed-ledger ${project.parent.version} test - ${project.groupId} pulsar-docs-tools @@ -63,57 +57,48 @@ - org.apache.commons commons-lang3 - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.inject jersey-hk2 - com.google.code.gson gson - io.swagger swagger-core provided - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - + org.eclipse.jetty.websocket websocket-api ${jetty.version} - - + org.eclipse.jetty.websocket websocket-server ${jetty.version} - + org.eclipse.jetty.websocket javax-websocket-client-impl @@ -128,9 +113,7 @@ org.hdrhistogram HdrHistogram - - @@ -150,7 +133,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -168,7 +150,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-websocket/src/main/resources/findbugsExclude.xml b/pulsar-websocket/src/main/resources/findbugsExclude.xml index c96e63cdfccee..e8f324925188b 100644 --- a/pulsar-websocket/src/main/resources/findbugsExclude.xml +++ b/pulsar-websocket/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + @@ -204,4 +204,4 @@ - \ No newline at end of file + diff --git a/src/assembly-source-package.xml b/src/assembly-source-package.xml index 00f00bfe3a5c7..bae52588edcf6 100644 --- a/src/assembly-source-package.xml +++ b/src/assembly-source-package.xml @@ -31,7 +31,7 @@ . - + true @@ -39,14 +39,11 @@ src/*.py docker/pulsar/scripts/*.sh docker/pulsar/scripts/*.py - data/** logs/** - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?maven-eclipse\.xml] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.project] @@ -72,12 +68,10 @@ %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.externalToolBuilders(/.*)?] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.deployables(/.*)?] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.wtpmodules(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?cobertura\.ser] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.versionsBackup] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?dependency-reduced-pom\.xml] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.releaseBackup] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties] @@ -86,7 +80,7 @@ ${project.build.directory}/maven-shared-archive-resources/META-INF - + src diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh index 3a6d266345f30..9c6a4d3223a70 100755 --- a/src/check-binary-license.sh +++ b/src/check-binary-license.sh @@ -61,7 +61,7 @@ EXIT=0 # Check all bundled jars are mentioned in LICENSE for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi @@ -104,7 +104,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi diff --git a/src/findbugs-exclude.xml b/src/findbugs-exclude.xml index 8f289b83a7be1..de4d3f7a93371 100644 --- a/src/findbugs-exclude.xml +++ b/src/findbugs-exclude.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/src/idea-code-style.xml b/src/idea-code-style.xml index 810dee34a3494..b49c523c322b2 100644 --- a/src/idea-code-style.xml +++ b/src/idea-code-style.xml @@ -1,3 +1,4 @@ + - - - - - \ No newline at end of file + diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 345be8f4d2c06..151752c98f7a1 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -1,4 +1,4 @@ - + - - + + file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-28164 - file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-29425 - file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-34429 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-28164 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-29425 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-34429 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-28164 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-29425 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-34429 - - file name: debezium-connector-postgres-1.7.2.Final.jar - ]]> + 69c1edfa7d89531af511fcd07e8516fa450f746a CVE-2021-23214 - - - + - file name: mariadb-java-client-2.7.5.jar - ]]> + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 CVE-2022-27376 CVE-2022-27377 @@ -162,43 +159,36 @@ CVE-2022-27386 CVE-2022-27387 - - file name: google-http-client-gson-1.41.0.jar - ]]> + 1a754a5dd672218a2ac667d7ff2b28df7a5a240e CVE-2022-25647 - commons-net is not used at all and therefore commons-net vulnerability CVE-2021-37533 is a false positive. CVE-2021-37533 - fredsmith utils library is not used at all. CVE-2021-4277 is a false positive. CVE-2021-4277 - It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-25194 is a false positive. CVE-2023-25194 - It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-34917 is a false positive. CVE-2022-34917 - yaml_project is not used at all. Any CVEs reported for yaml_project are false positives. cpe:/a:yaml_project:yaml - flat_project is not used at all. cpe:/a:flat_project:flat - \ No newline at end of file + diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index d5ddc28e884cb..31d48651a380e 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -1,4 +1,4 @@ - + - - - Ignore netty CVEs in GRPC shaded Netty. - .*grpc-netty-shaded.* - cpe:/a:netty:netty - - - Suppress all pulsar-presto-distribution vulnerabilities - .*pulsar-presto-distribution-.* - .* - - - Suppress libthrift-0.12.0.jar vulnerabilities - org.apache.thrift:libthrift:0.12.0 - .* - - - - - + + Ignore netty CVEs in GRPC shaded Netty. + .*grpc-netty-shaded.* + cpe:/a:netty:netty + + + Suppress all pulsar-presto-distribution vulnerabilities + .*pulsar-presto-distribution-.* + .* + + + Suppress libthrift-0.12.0.jar vulnerabilities + org.apache.thrift:libthrift:0.12.0 + .* + + + + file name: msgpack-core-0.9.0.jar - ]]> - 87d9ce0b22de48428fa32bb8ad476e18b6969548 - CVE-2022-41719 - - - - - + 87d9ce0b22de48428fa32bb8ad476e18b6969548 + CVE-2022-41719 + + + + file name: elasticsearch-java-8.1.0.jar CVE-2022-23712 is only related to Elastic server. - ]]> - edf5be04cbc2eafc51540ba33f9536e788b9d60b - CVE-2022-23712 - - - + edf5be04cbc2eafc51540ba33f9536e788b9d60b + CVE-2022-23712 + + + file name: elasticsearch-rest-client-8.1.0.jar CVE-2022-23712 is only related to Elastic server. - ]]> - 10e7aa09f10955a074c0a574cb699344d2745df1 - CVE-2022-23712 - - - - - + 10e7aa09f10955a074c0a574cb699344d2745df1 + CVE-2022-23712 + + + + file name: kotlin-stdlib-common-1.4.32.jar - ]]> - ef50bfa2c0491a11dcc35d9822edbfd6170e1ea2 - cpe:/a:jetbrains:kotlin - - - + ef50bfa2c0491a11dcc35d9822edbfd6170e1ea2 + cpe:/a:jetbrains:kotlin + + + file name: kotlin-stdlib-jdk7-1.4.32.jar - ]]> - 3546900a3ebff0c43f31190baf87a9220e37b7ea - CVE-2022-24329 - - - + 3546900a3ebff0c43f31190baf87a9220e37b7ea + CVE-2022-24329 + + + file name: kotlin-stdlib-jdk8-1.4.32.jar - ]]> - 3302f9ec8a5c1ed220781dbd37770072549bd333 - CVE-2022-24329 - - - + 3302f9ec8a5c1ed220781dbd37770072549bd333 + CVE-2022-24329 + + + file name: kotlin-stdlib-1.4.32.jar - ]]> - 461367948840adbb0839c51d91ed74ef4a9ccb52 - CVE-2022-24329 - - - - - + 461367948840adbb0839c51d91ed74ef4a9ccb52 + CVE-2022-24329 + + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2018-10237 - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2018-10237 + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2020-8908 - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2020-8908 + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:32.1.1) CVE cannot take effect. Already covered by PR https://github.com/apache/pulsar/pull/20699 - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2023-2976 - - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2023-2976 + + + + file name: avro-1.10.2.jar - ]]> - a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 - CVE-2021-43045 - - - + a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 + CVE-2021-43045 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14668 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14668 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14669 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14669 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14670 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14670 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14671 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14671 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14672 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14672 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-15024 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-15024 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-16535 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-16535 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-18657 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-18657 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2021-25263 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2021-25263 + + + file name: clickhouse-jdbc-0.4.6-all.jar (shaded: com.google.guava:guava:32.1.1) CVE cannot take effect. Already covered by PR https://github.com/apache/pulsar/pull/20699 - ]]> - d3b929509399a698915b24ff47db781d0c526760 - CVE-2023-2976 - - - + d3b929509399a698915b24ff47db781d0c526760 + CVE-2023-2976 + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - cpe:/a:qos:logback - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + cpe:/a:qos:logback + + + file name: rocketmq-acl-4.5.2.jar - ]]> - 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd - cpe:/a:apache:rocketmq - - - - ^pkg:maven/org\.springframework/spring.*$ - CVE-2016-1000027 - - - + 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd + cpe:/a:apache:rocketmq + + + Ignored since we are not vulnerable + ^pkg:maven/org\.springframework/spring.*$ + CVE-2016-1000027 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - cpe:/a:qos:logback - - - + d90276fff414f06cb375f2057f6778cd63c6082f + cpe:/a:qos:logback + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - CVE-2017-5929 - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2017-5929 - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2021-42550 - - - - Ignore etdc CVEs in jetcd - .*jetcd.* - cpe:/a:etcd:etcd - - - Ignore etdc CVEs in jetcd - .*jetcd.* - cpe:/a:redhat:etcd - - - Ignore grpc CVEs in jetcd - .*jetcd-grpc.* - cpe:/a:grpc:grpc - - - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2021-42550 + + + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:etcd:etcd + + + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:redhat:etcd + + + Ignore grpc CVEs in jetcd + .*jetcd-grpc.* + cpe:/a:grpc:grpc + + + + file name: bc-fips-1.0.2.jar - ]]> - 4fb5db5f03d00f6a94e43b78d097978190e4abb2 - CVE-2020-26939 - - - + 4fb5db5f03d00f6a94e43b78d097978190e4abb2 + CVE-2020-26939 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-15522 - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-15522 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-26939 - - - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-26939 + + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2016-0738 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2016-0738 + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2017-16613 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2017-16613 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-14432 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-14432 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-20170 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-20170 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12689 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12689 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12690 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12690 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12691 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12691 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12692 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12692 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2021-3563 - - - - - + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:pulsar - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:pulsar + + + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:solr - - - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:solr + + + + file name: debezium-connector-mysql-1.9.7.Final.jar - ]]> - 74c623b4a9b231e2f0e8f6053b38abd3bc487ce2 - CVE-2017-15945 - - - + 74c623b4a9b231e2f0e8f6053b38abd3bc487ce2 + CVE-2017-15945 + + + file name: mysql-binlog-connector-java-0.27.2.jar - ]]> - 23294cd730e29c00b8ddfbde517dfc955aba090f - CVE-2017-15945 - - - + 23294cd730e29c00b8ddfbde517dfc955aba090f + CVE-2017-15945 + + + file name: debezium-connector-postgres-1.9.7.Final.jar - ]]> - 300ff0bbf795643e914b7c8a6d6ba812a8354d62 - CVE-2015-0241 - CVE-2015-0242 - CVE-2015-0243 - CVE-2015-0244 - CVE-2015-3166 - CVE-2015-3167 - CVE-2016-0766 - CVE-2016-0768 - CVE-2016-0773 - CVE-2016-5423 - CVE-2016-5424 - CVE-2016-7048 - CVE-2017-14798 - CVE-2017-7484 - CVE-2018-1115 - CVE-2019-10127 - CVE-2019-10128 - CVE-2019-10210 - CVE-2019-10211 - CVE-2020-25694 - CVE-2020-25695 - CVE-2021-23214 - - - + 300ff0bbf795643e914b7c8a6d6ba812a8354d62 + CVE-2015-0241 + CVE-2015-0242 + CVE-2015-0243 + CVE-2015-0244 + CVE-2015-3166 + CVE-2015-3167 + CVE-2016-0766 + CVE-2016-0768 + CVE-2016-0773 + CVE-2016-5423 + CVE-2016-5424 + CVE-2016-7048 + CVE-2017-14798 + CVE-2017-7484 + CVE-2018-1115 + CVE-2019-10127 + CVE-2019-10128 + CVE-2019-10210 + CVE-2019-10211 + CVE-2020-25694 + CVE-2020-25695 + CVE-2021-23214 + + + file name: protostream-types-4.4.1.Final.jar - ]]> - 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 - CVE-2016-0750 - CVE-2017-15089 - CVE-2017-2638 - CVE-2019-10158 - CVE-2019-10174 - CVE-2020-25711 - - - + 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 + CVE-2016-0750 + CVE-2017-15089 + CVE-2017-2638 + CVE-2019-10158 + CVE-2019-10174 + CVE-2020-25711 + + + file name: mariadb-java-client-2.7.5.jar - ]]> - 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 - CVE-2020-28912 - CVE-2021-46669 - CVE-2021-46666 - CVE-2021-46667 - - - - + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 + CVE-2020-28912 + CVE-2021-46669 + CVE-2021-46666 + CVE-2021-46667 + + + + file name: cassandra-driver-core-3.11.2.jar - ]]> - e0aad9f8611e710b9a0ce49747f7465ce07d8404 - CVE-2020-17516 - CVE-2021-44521 - - - + e0aad9f8611e710b9a0ce49747f7465ce07d8404 + CVE-2020-17516 + CVE-2021-44521 + + + The vulnerable method is deprecated in Guava, but isn't removed. It's necessary to suppress this CVE. See https://github.com/google/guava/issues/4011 - ]]> - CVE-2020-8908 - - - + CVE-2020-8908 + + + This is a false positive in jackson-databind. See https://github.com/FasterXML/jackson-databind/issues/3972#issuecomment-1596604021 - ]]> - CVE-2023-35116 - + + CVE-2023-35116 + diff --git a/src/settings.xml b/src/settings.xml index 80ec2f40620e7..8fc2f13a37a59 100644 --- a/src/settings.xml +++ b/src/settings.xml @@ -1,3 +1,4 @@ + apache.releases.https @@ -34,7 +34,6 @@ ${env.APACHE_PASSWORD} - apache diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index c42a75397d0ca..414d37b217392 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - structured-event-log Structured event logger - org.slf4j slf4j-api - org.apache.logging.log4j @@ -60,7 +56,6 @@ test - @@ -78,5 +73,4 @@ - diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 30a29c7e3f9c4..54c705e17f190 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - pulsar - org.apache.pulsar + io.streamnative 3.1.0-SNAPSHOT - testmocks jar Pulsar Test Mocks - - org.apache.zookeeper zookeeper @@ -52,29 +48,23 @@ - org.apache.bookkeeper bookkeeper-server - org.apache.commons commons-lang3 - org.testng testng - org.objenesis objenesis - - @@ -92,5 +82,4 @@ - diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index eacf4623edff5..77214fe9de624 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.1.0-SNAPSHOT - - - bc_2_0_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.0-rc1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.1.0-SNAPSHOT + + bc_2_0_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.0-rc1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_0/src/test/resources/pulsar.xml b/tests/bc_2_0_0/src/test/resources/pulsar.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_0_0/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_0/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index d624c66170d56..5c3e244d0b41f 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.1.0-SNAPSHOT - - - bc_2_0_1 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.1.0-SNAPSHOT + + bc_2_0_1 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_1/src/test/resources/pulsar.xml b/tests/bc_2_0_1/src/test/resources/pulsar.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_0_1/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_1/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 2cd8e7f786809..b53fbe28ba865 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -1,4 +1,4 @@ - + - - - org.apache.pulsar.tests - tests-parent - 3.1.0-SNAPSHOT - - 4.0.0 - - bc_2_6_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 - - - - - com.google.code.gson - gson - test - - - - org.apache.pulsar - pulsar-client - 2.6.0 - test - - - - org.testcontainers - testcontainers - test - - - - - + + + io.streamnative.tests + tests-parent + 3.1.0-SNAPSHOT + + 4.0.0 + bc_2_6_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 + + + com.google.code.gson + gson + test + + + org.apache.pulsar + pulsar-client + 2.6.0 + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/backwards-client.xml - - 1 - - - - - - - - + false + + src/test/resources/backwards-client.xml + + 1 + + + + + + diff --git a/tests/bc_2_6_0/src/test/resources/backwards-client.xml b/tests/bc_2_6_0/src/test/resources/backwards-client.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_6_0/src/test/resources/backwards-client.xml +++ b/tests/bc_2_6_0/src/test/resources/backwards-client.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 0afb21c6a03a2..0197542a424c5 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.1.0-SNAPSHOT @@ -30,12 +30,12 @@ Apache Pulsar :: Tests :: Docker Images :: Java Test Functions - org.apache.pulsar + io.streamnative pulsar-io-core ${project.version} - org.apache.pulsar + io.streamnative pulsar-functions-api ${project.version} @@ -59,7 +59,6 @@ jar - docker @@ -68,15 +67,13 @@ integrationTests - - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} - @@ -91,7 +88,7 @@ - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples @@ -102,5 +99,4 @@ - diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 84b02ad137cfe..033bf22f3d6e7 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.1.0-SNAPSHOT @@ -29,7 +29,6 @@ java-test-image Apache Pulsar :: Tests :: Docker Images :: Java Test Image pom - docker @@ -45,12 +44,12 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin @@ -78,7 +77,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -87,7 +86,7 @@ java-test-functions.jar - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index 16b0ec1911793..4b1817453bb79 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.1.0-SNAPSHOT @@ -31,7 +31,7 @@ jar - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} provided @@ -45,5 +45,4 @@ - diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 2aac3b14be2ea..843932076d4cf 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.1.0-SNAPSHOT @@ -29,7 +29,6 @@ latest-version-image Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing pom - docker @@ -40,17 +39,17 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar.tests + io.streamnative.tests java-test-plugins ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-all-docker-image ${project.parent.version} pom @@ -77,7 +76,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -86,7 +85,7 @@ java-test-functions.jar - org.apache.pulsar.tests + io.streamnative.tests java-test-plugins ${project.parent.version} jar diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index ce45569c2f1a9..a0426a7d6e925 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent 3.1.0-SNAPSHOT diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 40e97f5a6ebc5..9af50045a52d8 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent 3.1.0-SNAPSHOT - integration jar Apache Pulsar :: Tests :: Integration - pulsar.xml 4.1.2 - com.google.code.gson @@ -44,37 +40,37 @@ test - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} test - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} test - org.apache.pulsar + io.streamnative pulsar-common ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-original ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-admin-original ${project.version} test - org.apache.pulsar + io.streamnative managed-ledger ${project.version} test @@ -96,31 +92,27 @@ test - org.apache.pulsar + io.streamnative pulsar-io-kafka ${project.version} test - org.testcontainers mysql test - org.postgresql postgresql ${postgresql-jdbc.version} runtime - org.testcontainers postgresql test - com.github.docker-java docker-java-core @@ -131,72 +123,60 @@ bcpkix-jdk18on test - - org.apache.pulsar + io.streamnative pulsar-io-jdbc-postgres ${project.version} test - com.fasterxml.jackson.core jackson-databind test - com.fasterxml.jackson.dataformat jackson-dataformat-yaml test - org.opensearch.client opensearch-rest-high-level-client test - co.elastic.clients elasticsearch-java test - org.testcontainers elasticsearch test - - com.rabbitmq amqp-client ${rabbitmq-client.version} test - joda-time joda-time test - io.trino trino-jdbc ${trino.version} test - org.awaitility awaitility test - - + org.testcontainers localstack @@ -213,7 +193,6 @@ aws-java-sdk-core test - org.testcontainers @@ -226,9 +205,7 @@ ${mongo-reactivestreams.version} test - - @@ -269,7 +246,6 @@ - integrationTests diff --git a/tests/integration/src/test/resources/pulsar-auth.xml b/tests/integration/src/test/resources/pulsar-auth.xml index 81d2c1361a269..3647730dee201 100644 --- a/tests/integration/src/test/resources/pulsar-auth.xml +++ b/tests/integration/src/test/resources/pulsar-auth.xml @@ -1,3 +1,4 @@ + - + - - + + - + diff --git a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml index b50af74ed17f0..b0f55f3e04b16 100644 --- a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml +++ b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-cli.xml b/tests/integration/src/test/resources/pulsar-cli.xml index af55aca8a0098..1243dc7e485d7 100644 --- a/tests/integration/src/test/resources/pulsar-cli.xml +++ b/tests/integration/src/test/resources/pulsar-cli.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-function.xml b/tests/integration/src/test/resources/pulsar-function.xml index a18a65ff2e16d..03a8d83997a3c 100644 --- a/tests/integration/src/test/resources/pulsar-function.xml +++ b/tests/integration/src/test/resources/pulsar-function.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-ora-source.xml b/tests/integration/src/test/resources/pulsar-io-ora-source.xml index 1c5bb5faf677c..0a9927015be86 100644 --- a/tests/integration/src/test/resources/pulsar-io-ora-source.xml +++ b/tests/integration/src/test/resources/pulsar-io-ora-source.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sinks.xml b/tests/integration/src/test/resources/pulsar-io-sinks.xml index 614c371f3e52c..046f977f3e13b 100644 --- a/tests/integration/src/test/resources/pulsar-io-sinks.xml +++ b/tests/integration/src/test/resources/pulsar-io-sinks.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml index 636b3e479195f..c995a7df249dd 100644 --- a/tests/integration/src/test/resources/pulsar-io-sources.xml +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-loadbalance.xml b/tests/integration/src/test/resources/pulsar-loadbalance.xml index dfc4536e25592..d28dd1e0da7ee 100644 --- a/tests/integration/src/test/resources/pulsar-loadbalance.xml +++ b/tests/integration/src/test/resources/pulsar-loadbalance.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index cfbdb22587034..f8813b614f5cb 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-plugin.xml b/tests/integration/src/test/resources/pulsar-plugin.xml index f88b67256e547..06fa8ee628c0f 100644 --- a/tests/integration/src/test/resources/pulsar-plugin.xml +++ b/tests/integration/src/test/resources/pulsar-plugin.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-process.xml b/tests/integration/src/test/resources/pulsar-process.xml index 8e5258d30624c..a6f1ac468d954 100644 --- a/tests/integration/src/test/resources/pulsar-process.xml +++ b/tests/integration/src/test/resources/pulsar-process.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-proxy.xml b/tests/integration/src/test/resources/pulsar-proxy.xml index ae6f13810535c..fabfb54d781a3 100644 --- a/tests/integration/src/test/resources/pulsar-proxy.xml +++ b/tests/integration/src/test/resources/pulsar-proxy.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-python.xml b/tests/integration/src/test/resources/pulsar-python.xml index a5faa6389e0f1..365768083d21b 100644 --- a/tests/integration/src/test/resources/pulsar-python.xml +++ b/tests/integration/src/test/resources/pulsar-python.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml index e07fdf2b2d86f..595bb8fda5521 100644 --- a/tests/integration/src/test/resources/pulsar-schema.xml +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -1,3 +1,4 @@ + - + - - - - - - - + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-semantics.xml b/tests/integration/src/test/resources/pulsar-semantics.xml index 5b5402af4623b..69e171931f0e4 100644 --- a/tests/integration/src/test/resources/pulsar-semantics.xml +++ b/tests/integration/src/test/resources/pulsar-semantics.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-sql.xml b/tests/integration/src/test/resources/pulsar-sql.xml index 1ab4d479ad041..a7e0995f1f2d0 100644 --- a/tests/integration/src/test/resources/pulsar-sql.xml +++ b/tests/integration/src/test/resources/pulsar-sql.xml @@ -1,3 +1,4 @@ + - + - - - - - - - + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-standalone.xml b/tests/integration/src/test/resources/pulsar-standalone.xml index d8892c0746181..45d79a72821b5 100644 --- a/tests/integration/src/test/resources/pulsar-standalone.xml +++ b/tests/integration/src/test/resources/pulsar-standalone.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-thread.xml b/tests/integration/src/test/resources/pulsar-thread.xml index cf3da15e8e1ad..dc1b8e3b3de15 100644 --- a/tests/integration/src/test/resources/pulsar-thread.xml +++ b/tests/integration/src/test/resources/pulsar-thread.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-tls.xml b/tests/integration/src/test/resources/pulsar-tls.xml index 153d14b62a725..88e90c43107fb 100644 --- a/tests/integration/src/test/resources/pulsar-tls.xml +++ b/tests/integration/src/test/resources/pulsar-tls.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-transaction.xml b/tests/integration/src/test/resources/pulsar-transaction.xml index 72c375d000d8e..425f58ebda639 100644 --- a/tests/integration/src/test/resources/pulsar-transaction.xml +++ b/tests/integration/src/test/resources/pulsar-transaction.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-upgrade.xml b/tests/integration/src/test/resources/pulsar-upgrade.xml index a52db54753372..cb312b04ed7a3 100644 --- a/tests/integration/src/test/resources/pulsar-upgrade.xml +++ b/tests/integration/src/test/resources/pulsar-upgrade.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-websocket.xml b/tests/integration/src/test/resources/pulsar-websocket.xml index 87bf832d4e40a..e5023451518e0 100644 --- a/tests/integration/src/test/resources/pulsar-websocket.xml +++ b/tests/integration/src/test/resources/pulsar-websocket.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index d309a7695dd0b..ada31c2e5da8c 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/tiered-filesystem-storage.xml b/tests/integration/src/test/resources/tiered-filesystem-storage.xml index b14077594e0c9..c8cb694ea2d02 100644 --- a/tests/integration/src/test/resources/tiered-filesystem-storage.xml +++ b/tests/integration/src/test/resources/tiered-filesystem-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/tiered-jcloud-storage.xml b/tests/integration/src/test/resources/tiered-jcloud-storage.xml index f61f9f5fa1612..be699ba0a8680 100644 --- a/tests/integration/src/test/resources/tiered-jcloud-storage.xml +++ b/tests/integration/src/test/resources/tiered-jcloud-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/pom.xml b/tests/pom.xml index ecdff86a8b1b2..f03dcc35b4844 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT - org.apache.pulsar.tests + io.streamnative.tests tests-parent Apache Pulsar :: Tests @@ -50,7 +49,7 @@ skipIntegrationTests - + integrationTests diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index b0019d01b04ee..0cb5af8dcebee 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.1.0-SNAPSHOT - - - pulsar-client-admin-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test - - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin-api - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.1.0-SNAPSHOT + + pulsar-client-admin-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + io.streamnative + pulsar-client-admin-api + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml index d95dd95ae1775..2476755a95f7e 100644 --- a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index db11fec205a93..775e291cb0ae6 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.1.0-SNAPSHOT - - - pulsar-client-all-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test - - - - - org.apache.pulsar - pulsar-client-all - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.1.0-SNAPSHOT + + pulsar-client-all-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test + + + io.streamnative + pulsar-client-all + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + bouncy-castle-bc + ${project.version} + pkg + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml index 5c725f80eaea6..cfbc5ce59488b 100644 --- a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index cf5aff0291fb7..e332e5d59107c 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar.tests - tests-parent - 3.1.0-SNAPSHOT - - - pulsar-client-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Shade Test - - - - - org.apache.pulsar - pulsar-client - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.1.0-SNAPSHOT + + pulsar-client-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Shade Test + + + io.streamnative + pulsar-client + ${project.version} + test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml index 6132ad05f6ccf..e2ac6a9ee4f30 100644 --- a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 51986d17c3dbf..9d97a7e4dd548 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - tiered-storage-parent - 3.1.0-SNAPSHOT - .. - - - tiered-storage-file-system - Apache Pulsar :: Tiered Storage :: File System - - - ${project.groupId} - managed-ledger - ${project.version} - - - org.apache.hadoop - hadoop-common - ${hdfs-offload-version3} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hadoop - hadoop-hdfs-client - ${hdfs-offload-version3} - - - org.apache.avro - avro - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - - - - org.apache.avro - avro - ${avro.version} - - - - net.minidev - json-smart - ${json-smart.version} - - - com.google.protobuf - protobuf-java - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.hadoop - hadoop-minicluster - ${hdfs-offload-version3} - test - - - io.netty - netty-all - - - org.bouncycastle - * - - - - - - org.bouncycastle - bcpkix-jdk18on - test - - - + + 4.0.0 + + io.streamnative + tiered-storage-parent + 3.1.0-SNAPSHOT + .. + + tiered-storage-file-system + Apache Pulsar :: Tiered Storage :: File System + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.apache.hadoop + hadoop-common + ${hdfs-offload-version3} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hdfs-offload-version3} + + + org.apache.avro + avro + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + javax.servlet + servlet-api + + + + + + org.apache.avro + avro + ${avro.version} + + + + net.minidev + json-smart + ${json-smart.version} + + + com.google.protobuf + protobuf-java + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.hadoop + hadoop-minicluster + ${hdfs-offload-version3} + test + + io.netty - netty-codec-http - - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-alpn-conscrypt-server - test - - - org.eclipse.jetty - jetty-servlet - test - - - org.eclipse.jetty - jetty-util - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/tiered-storage/file-system/src/main/resources/findbugsExclude.xml b/tiered-storage/file-system/src/main/resources/findbugsExclude.xml index 051f0e68e257a..1b8cc1ac799ca 100644 --- a/tiered-storage/file-system/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/file-system/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative tiered-storage-parent 3.1.0-SNAPSHOT .. - tiered-storage-jcloud Apache Pulsar :: Tiered Storage :: JCloud - ${project.groupId} managed-ledger ${project.version} - ${project.groupId} jclouds-shaded @@ -74,7 +70,6 @@ - org.apache.jclouds jclouds-allblobstore @@ -85,12 +80,10 @@ com.amazonaws aws-java-sdk-core - com.amazonaws aws-java-sdk-sts - ${project.groupId} testmocks @@ -103,7 +96,6 @@ ${jclouds.version} provided - javax.xml.bind jaxb-api @@ -115,13 +107,11 @@ runtime - com.sun.activation javax.activation runtime - @@ -129,7 +119,6 @@ org.apache.nifi nifi-nar-maven-plugin - com.github.spotbugs spotbugs-maven-plugin diff --git a/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml b/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml index a8a9b9aae925f..c6ad70823a5ac 100644 --- a/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.1.0-SNAPSHOT .. - tiered-storage-parent Apache Pulsar :: Tiered Storage :: Parent - ${project.version} - jcloud file-system