Merge branch 'master' into Batch-subscription-should-be-stoppable-des…

…pite-of-pending-message-retries

.github/workflows/checkstyle.yml

@@ -10,20 +10,22 @@ jobs:
   checkstyle:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: reviewdog/action-setup@v1
         with:
           reviewdog_version: latest
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 17
           distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
       - name: Run check style
         # ignore lengthy console setup tasks
-        run: ./gradlew --continue clean checkstyleMain checkstyleTest checkstyleIntegration checkstyleJmh -PmaxCheckstyleWarnings=0 -x attachHermesConsole -x prepareIndexTemplate
+        run: ./gradlew --continue clean checkstyleMain checkstyleTest checkstyleIntegrationTest checkstyleSlowIntegrationTest checkstyleJmh -PmaxCheckstyleWarnings=0 -x attachHermesConsole -x prepareIndexTemplate
       - name: Run reviewdog
         if: ${{ success() || failure() }}
         env:

.github/workflows/ci-console.yml

@@ -9,14 +9,14 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./hermes-console-vue
+        working-directory: ./hermes-console
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
       - name: Run linter
         run: yarn && yarn lint
       - name: Run frontend tests

.github/workflows/ci.yml

@@ -14,27 +14,22 @@ jobs:
         tasks: [
           {alias: "unitTests", name: "check"},
           {alias: "integrationTests", name: "integrationTest"},
+          {alias: "slowIntegrationTests", name: "slowIntegrationTest"},
           {alias: "benchmark", name: "jmh -Pjmh.iterations=1 -Pjmh.timeOnIteration=5s -Pjmh.warmupIterations=0"}
         ]
       fail-fast: false
     name: ${{ matrix.tasks.alias }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 17
           distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
       - name: Build with Gradle
         run: ./gradlew assemble
       - name: Run task with Gradle

.github/workflows/codeql-analysis.yml

@@ -35,17 +35,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up JDK 17
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         java-version: 17
         distribution: 'temurin'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -70,4 +70,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/gradle-wrapper-validation.yml

@@ -12,5 +12,5 @@ jobs:
     name: "Validation"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: gradle/wrapper-validation-action@v1

.github/workflows/markdown-links-check.yml

@@ -12,7 +12,7 @@ jobs:
   check-links:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:
           use-quiet-mode: 'yes'

.github/workflows/release.yml

@@ -16,15 +16,16 @@ jobs:
     environment: ci
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: gradle/wrapper-validation-action@v1
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
       - name: Release
         if: github.ref == 'refs/heads/master'
         run: ./gradlew release -Prelease.customPassword=${GITHUB_TOKEN} -Prelease.customUsername=${GITHUB_ACTOR} -Prelease.forceVersion=${FORCE_VERSION}

.github/workflows/test_report.yml

@@ -8,14 +8,14 @@ jobs:
     name: "Test report"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Grant execute permission for report downloader
         run: chmod +x ./.github/scripts/download_reports.sh
       - name: Download past reports
         run: ./.github/scripts/download_reports.sh
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Aggregate reports

CONTRIBUTING.md

@@ -27,5 +27,5 @@ You can also use other *magic words* from [GitHub handbook](https://help.github.
 
 * use `spock` when writing new unit tests in all modules
 * when changing old tests use your best judgement as to when rewrite them to `spock`
-* use `TestNG` with defined environment in `integration` module
+* use `JUnit5` with defined environment in `integration-tests` module
 * prepend configuration options with module name, i.e. `frontend.` or `consumer.` when it applies to single module

build.gradle

@@ -48,27 +48,27 @@ allprojects {
     targetCompatibility = JavaVersion.VERSION_17
 
     project.ext.versions = [
-            kafka             : '2.8.2',
+            kafka             : '3.6.2',
             guava             : '23.0',
             jackson           : '2.15.2',
             jersey            : '3.1.2',
-            jetty             : '11.0.15',
+            jetty             : '12.0.7',
             curator           : '5.4.0',
             dropwizard_metrics: '4.1.0',
             micrometer_metrics: '1.11.1',
-            wiremock          : '3.0.1',
+            wiremock          : '3.3.1',
             spock             : '2.4-M1-groovy-4.0',
             groovy            : '4.0.12',
             avro              : '1.9.1',
             json2avro         : '0.2.14',
             okhttp            : '3.9.1',
             undertow          : '2.0.29.Final',
-            spring_web        : '6.0.8',
+            spring_web        : '6.1.2',
             failsafe          : '2.3.1',
-            junit_jupiter     : '5.8.2',
-            testcontainers    : '1.18.1',
-            spring            : '3.0.6',
-            assertj           : '3.22.0'
+            junit_jupiter     : '5.9.1',
+            testcontainers    : '1.19.8',
+            spring            : '3.2.1',
+            assertj           : '3.24.2'
     ]
 
     repositories {
@@ -113,7 +113,7 @@ allprojects {
 }
 
 
-configure(subprojects - project(':integration')) {
+configure(subprojects - project(':integration-tests')) {
     apply plugin: 'jacoco'
     apply plugin: 'maven-publish'
     apply plugin: 'signing'
@@ -123,6 +123,8 @@ configure(subprojects - project(':integration')) {
         withSourcesJar()
     }
 
+    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
+
     publishing {
         publications {
             mavenJava(MavenPublication) {

config/checkstyle/checkstyle.xml

@@ -46,7 +46,7 @@
     -->
     <module name="LineLength">
         <property name="fileExtensions" value="java"/>
-        <property name="max" value="140"/>
+        <property name="max" value="400"/>
         <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
     </module>
 

config/checkstyle/suppressions.xml

@@ -36,6 +36,9 @@
     <suppress checks="VisibilityModifier"
               files="[/\\]src[/\\](test|integration)[/\\]java[/\\]"
             />
+    <suppress checks="VariableDeclarationUsageDistance"
+              files="[/\\]src[/\\](test|slowIntegrationTest)[/\\]java[/\\]"
+    />
 
     <!-- Build dirs -->
     <suppress checks="[a-zA-Z0-9]*"
@@ -44,4 +47,10 @@
     <!-- Classes repackaged from external libraries -->
     <suppress checks="[a-zA-Z0-9]*"
               files="(DirectBufferPool|LinkedHashSetBlockingQueue)\.java" />
+
+    <suppress checks="MethodName"
+              files="ChronicleMapMessageRepositoryTest\.java" />
+
+    <suppress checks="LocalVariableName"
+              files="ChronicleMapMessageRepositoryTest\.java" />
 </suppressions>

docs/docs/configuration/console.md

@@ -15,14 +15,12 @@ dashboard.docs    | link to documentation, available on Console home page
 
 ## Metric Store integration
 
-Hermes Console can be integrated with Metric Store. This means, that metrics shown in Console can link to actual graphs
-plotted by Metric Store. At the moment only Graphite is supported.
-
-Option                  | Description
------------------------ | ---------------------------------------------------------------------------
-metrics.type            | type of metrics storage to link to (currently only `graphite` is supported)
-metrics.graphite.url    | URL to graphite
-metrics.graphite.prefix | prefix to graphite metrics
+Hermes console could have a button on the topics and subscriptions view that takes you to a dashboard with metrics.
+In order to make it work you have to provide an implementation of `pl.allegro.tech.hermes.management.domain.MetricsDashboardUrlService`.
+
+ Option                              | Description                                                                      
+-------------------------------------|--------------------------------------------------------------------------------------
+ metrics.fetchingDashboardUrlEnabled | enable fetching dashboard url from hermes-management and show the referring UI button 
 
 ## Authorization
 

docs/docs/configuration/consumers-tuning.md

@@ -2,18 +2,18 @@
 
 ## HTTP Sender
 
-Option                                               | Description                                                 | Default value
----------------------------------------------------- | ----------------------------------------------------------- | -------------
-consumer.http-client.serial.http1.threadPoolSize                  | size of thread pool for sender threads (global)             | 30
-consumer.http-client.serial.http1.maxConnectionsPerDestination    | max connections per remote host                             | 100
+| Option                                                         | Description                                     | Default value |
+|----------------------------------------------------------------|-------------------------------------------------|---------------|
+| consumer.http-client.serial.http1.threadPoolSize               | size of thread pool for sender threads (global) | 30            |
+| consumer.http-client.serial.http1.maxConnectionsPerDestination | max connections per remote host                 | 100           |
 
 ## Consumers core
 
-Option                        | Description                                                              | Default value
------------------------------ | ------------------------------------------------------------------------ | -------------
-consumer.commit.offset.period | interval between committing offsets to Kafka                             | 60s
-consumer.threadPoolSize       | thread pool for threads involved in consuming, 1 thread per subscription | 500
-consumer.serialConsumer.inflightSize         | how many messages can be kept in send queue, per subscription            | 100
+| Option                               | Description                                                              | Default value |
+|--------------------------------------|--------------------------------------------------------------------------|---------------|
+| consumer.commit.offset.period        | interval between committing offsets to Kafka                             | 60s           |
+| consumer.threadPoolSize              | thread pool for threads involved in consuming, 1 thread per subscription | 500           |
+| consumer.serialConsumer.inflightSize | how many messages can be kept in send queue, per subscription            | 100           |
 
 ## Workload constraints management
 
@@ -26,10 +26,10 @@ subscriptions assigned to itself.
 
 These numbers can be configured:
 
-Option                                              | Description                               | Default value
---------------------------------------------------- | ----------------------------------------- | ---------------------
-consumer.workload.consumersPerSubscription          | Number of consumers to which the subscription will be assigned. If this value is greater than the number of available consumers, Hermes will assign the subscription to all available consumers. | 2
-consumer.workload.maxSubscriptionsPerConsumer       | The maximum number of subscriptions assigned to a single consumer. If all consumers have the maximum number of subscriptions assigned, a new subscription will not be activated until a new consumer is added or another subscription is unassigned. | 200
+| Option                                        | Description                                                                                                                                                                                                                                          | Default value |
+|-----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
+| consumer.workload.consumersPerSubscription    | Number of consumers to which the subscription will be assigned. If this value is greater than the number of available consumers, Hermes will assign the subscription to all available consumers.                                                     | 2             |
+| consumer.workload.maxSubscriptionsPerConsumer | The maximum number of subscriptions assigned to a single consumer. If all consumers have the maximum number of subscriptions assigned, a new subscription will not be activated until a new consumer is added or another subscription is unassigned. | 200           |
 
 Additionally, Hermes allows to configure the property `consumer.workload.consumersPerSubscription` for specific
 topics or subscriptions in the runtime via REST API.

docs/docs/user/publishing.md

@@ -168,3 +168,7 @@ buffered events.
 Default implementation uses [OpenHFT ChronicleMap](https://github.com/OpenHFT/Chronicle-Map) to persist unsent messages
 to disk. Map structure is continuously persisted to disk, as it is stored in offheap memory as
 [memory mapped file](https://en.wikipedia.org/wiki/Memory-mapped_file).
+
+## Partition assignment
+`Partition-Key` header can be used by publishers to specify Kafka `key` which will be used for partition assignment for a message. This will ensure 
+that all messages with given `Partition-Key` will be sent to the same Kafka partition.