diff --git a/.travis.yml b/.travis.yml index 612cd0d9..a6d04692 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ env: - ORG_GRADLE_PROJECT_sonatypeUsername=xxx - ORG_GRADLE_PROJECT_sonatypePassword=xxx - USER_NAME=rqueue + - REDIS_RUNNING=true + - CI_ENV=true cache: directories: @@ -30,6 +32,8 @@ before_script: - cd 9004 && redis-server ./redis.conf & - cd 9005 && redis-server ./redis.conf & - redis-cli --cluster create 127.0.0.1:9000 127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.0.0.1:9005 --cluster-replicas 1 --cluster-yes + - df -h + - lscpu jobs: include: diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f7deb30..37113d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ **NOTE**: The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.0] - 16-Sep-2020 +### Added +* Allow application to provide message id while enqueuing messages +* Unique message enqueue +* Api to check if message was enqueued or not +* Api to delete single message +* Proxy for outbound http connection +* Enqueue list of objects and process them, like batch-processing + +Fixes: +* Registered queues should not be deleted when used in producer mode ## [2.0.4] - 2-Aug-2020 @@ -92,3 +103,4 @@ [2.0.1]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/2.0.1-RELEASE [2.0.2]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/2.0.2-RELEASE [2.0.4]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/2.0.4-RELEASE +[2.1.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/2.1.0-RELEASE diff --git a/README.md b/README.md index 4e2bf8ac..986aacf8 100644 --- a/README.md +++ b/README.md @@ -13,20 +13,22 @@ ## Features -* A message can be delayed for an arbitrary period or delivered immediately. -* Multiple messages can be consumed in parallel by different workers. -* Message delivery: It's guaranteed that a message is consumed **at least once**. (Message would be consumed by a worker more than once due to the failure in the underlying worker/restart-process etc, otherwise exactly one delivery) -* Support Redis cluster -* Queue metrics -* Different Redis connection for application and worker -* Web interface for queue management and queue statistics -* Automatic message serialization and deserialization -* Queue concurrency -* Group level queue priority(weighted and strict) -* Sub queue priority(weighted and strict) -* Task execution back off, exponential and fixed back off (default fixed back off) -* Callbacks for different actions -* Events 1. Bootstrap event 2. Task execution event. +* **Message Scheduling** : A message can be scheduled for any arbitrary period +* **Competing Consumers** multiple messages can be consumed in parallel by different workers. +* **Message delivery**: It's guaranteed that a message is consumed **at least once**. (Message would be consumed by a worker more than once due to the failure in the underlying worker/restart-process etc, otherwise exactly one delivery) +* **Redis cluster** : Redis cluster can be used with driver. +* **Metrics** : In flight messages, waiting for consumption and delayed messages +* **Web interface**: a web interface to manage a queue and queue insights including latency +* **Automatic message serialization and deserialization** +* **Concurrency**: Concurrency of any queue can be configured +* **Queue Priority** : + * Group level queue priority(weighted and strict) + * Sub queue priority(weighted and strict) +* **Execution Backoff** : Exponential and fixed back off (default fixed back off) +* **Callbacks** : Callbacks for dead letter queue, discard etc +* **Events** 1. Bootstrap event 2. Task execution event. +* **Unique message** : Unique message processing for a queue based on the message id +* **Redis connection**: A different redis setup can be used for Rqueue ## Getting Started @@ -37,14 +39,14 @@ * Add dependency * Gradle ```groovy - implementation 'com.github.sonus21:rqueue-spring-boot-starter:2.0.4-RELEASE' + implementation 'com.github.sonus21:rqueue-spring-boot-starter:2.1.0-RELEASE' ``` * Maven ```xml com.github.sonus21 rqueue-spring-boot-starter - 2.0.4-RELEASE + 2.1.0-RELEASE ``` @@ -53,14 +55,14 @@ * Add Dependency * Gradle ```groovy - implementation 'com.github.sonus21:rqueue-spring:2.0.4-RELEASE' + implementation 'com.github.sonus21:rqueue-spring:2.1.0-RELEASE' ``` * Maven ```xml com.github.sonus21 rqueue-spring - 2.0.4-RELEASE + 2.1.0-RELEASE ``` diff --git a/build.gradle b/build.gradle index 9a540302..3eb379d0 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,7 @@ allprojects { apply plugin: 'signing' apply plugin: 'jacoco' apply plugin: 'nebula.optional-base' + sourceCompatibility = 1.8 targetCompatibility = 1.8 @@ -36,10 +37,9 @@ ext { sl4jVersion = '1.7.28' // testing - junitVersion = '4.12' - mockitoVersion = '3.1.0' + jupiterVersion = '5.5.0' + mockitoVersion = '3.5.0' hamcrestVersion = '2.2' - powerMockVersion = '2.0.5' jacocoVersion = '0.8.4' embeddedRedisVersion = '0.7.2' h2Version = '1.4.194' @@ -67,7 +67,7 @@ ext { subprojects { group = 'com.github.sonus21' - version = '2.0.4-RELEASE' + version = '2.1.0-RELEASE' dependencies { // https://mvnrepository.com/artifact/org.springframework/spring-messaging @@ -84,22 +84,21 @@ subprojects { testCompile group: 'ch.qos.logback', name: 'logback-core', version: "${logbackVersion}" // https://mvnrepository.com/artifact/ch.qos.logback/logback-classic testCompile group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" - testCompile group: 'junit', name: 'junit', version: "${junitVersion}" - // https://mvnrepository.com/artifact/org.mockito/mockito-core - testCompile group: 'org.mockito', name: 'mockito-core', version: "${mockitoVersion}" - // https://mvnrepository.com/artifact/org.hamcrest/hamcrest-library - testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrestVersion}" + testRuntime group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: "${jupiterVersion}" + testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: "${jupiterVersion}" + // https://mvnrepository.com/artifact/org.mockito/mockito-inline + testCompile group: 'org.mockito', name: 'mockito-inline', version: "${mockitoVersion}" + testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockitoVersion}" + // https://mvnrepository.com/artifact/org.hamcrest/hamcrest + testCompile group: 'org.hamcrest', name: 'hamcrest', version: "${hamcrestVersion}" // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "${lang3Version}" testCompile("org.springframework.boot:spring-boot-test:${springBootVersion}") - // https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 - testCompile group: 'org.powermock', name: 'powermock-module-junit4', version: "${powerMockVersion}" - - // https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 - testCompile group: 'org.powermock', name: 'powermock-api-mockito2', version: "${powerMockVersion}" + testCompile group: 'org.junit-pioneer', name: 'junit-pioneer', version: '0.9.0' configurations { all*.exclude module: 'spring-boot-starter-logging' + all*.exclude module: 'junit' } } } diff --git a/rqueue-common-test/build.gradle b/rqueue-common-test/build.gradle index 2850e3e7..56cd031a 100644 --- a/rqueue-common-test/build.gradle +++ b/rqueue-common-test/build.gradle @@ -23,7 +23,7 @@ dependencies { // https://mvnrepository.com/artifact/javax.annotation/javax.annotation-api compile group: 'javax.annotation', name: 'javax.annotation-api', version: "${javaxAnnotationVersion}" - compile group: 'junit', name: 'junit', version: "${junitVersion}" + compile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: "${jupiterVersion}" compile group: 'org.apache.commons', name: 'commons-lang3', version: "${lang3Version}" diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java index 72ecd2ec..d5548cef 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.test; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.sonus21.rqueue.annotation.RqueueListener; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; @@ -29,6 +30,8 @@ import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.test.service.ConsumedMessageService; import com.github.sonus21.rqueue.test.service.FailureManager; +import java.util.List; +import java.util.UUID; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -163,4 +166,13 @@ public void onMessageReservationRequestDeadLetterQueue(ReservationRequest reques log.info("ReservationRequest Dead Letter Queue{}", request); consumedMessageService.save(request, "reservation-request-dlq"); } + + @RqueueListener(value = "${list.email.queue.name}", active = "${list.email.queue.enabled}") + public void onMessageEmailList(List emailList) throws JsonProcessingException { + log.info("onMessageEmailList {}", emailList); + String consumedId = UUID.randomUUID().toString(); + for (Email email : emailList) { + consumedMessageService.save(email, consumedId); + } + } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java new file mode 100644 index 00000000..d001a1d6 --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java @@ -0,0 +1,139 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.application; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.sql.DataSource; +import lombok.AllArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.data.redis.connection.RedisNode; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import redis.embedded.RedisServer; + +public abstract class ApplicationBasicConfiguration { + private static final Logger monitorLogger = LoggerFactory.getLogger("monitor"); + protected RedisServer redisServer; + protected ExecutorService executorService; + protected List processes; + @Value("${mysql.db.name}") + protected String dbName; + @Value("${spring.redis.port}") + protected int redisPort; + @Value("${spring.redis.host}") + protected String redisHost; + @Value("${use.system.redis:false}") + protected boolean useSystemRedis; + @Value("${monitor.thread.count:0}") + protected int monitorThreads; + + protected void init() { + if (monitorThreads > 0) { + executorService = Executors.newFixedThreadPool(monitorThreads); + processes = new ArrayList<>(); + } + if (useSystemRedis) { + return; + } + if (redisServer == null) { + redisServer = new RedisServer(redisPort); + redisServer.start(); + } + } + + protected void destroy() { + if (redisServer != null) { + redisServer.stop(); + } + + if (processes != null) { + for (RProcess rProcess : processes) { + rProcess.process.destroy(); + monitorLogger.info("RedisNode {} ", rProcess.redisNode); + for (String line : rProcess.out) { + monitorLogger.info("{}", line); + } + } + } + if (executorService != null) { + executorService.shutdown(); + } + } + + protected void monitor(String host, int port) { + executorService.submit( + () -> { + try { + Process process = + Runtime.getRuntime() + .exec("redis-cli " + " -h " + host + " -p " + port + " monitor"); + List lines = new LinkedList<>(); + RProcess rProcess = new RProcess(process, new RedisNode(host, port), lines); + processes.add(rProcess); + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + String s; + while ((s = br.readLine()) != null) { + lines.add(s); + } + process.waitFor(); + } catch (Exception e) { + monitorLogger.error("Process call failed", e); + } + }); + } + + @Bean + public DataSource dataSource() { + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder.setType(EmbeddedDatabaseType.H2).setName(dbName).build(); + } + + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setGenerateDdl(true); + LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); + factory.setJpaVendorAdapter(vendorAdapter); + factory.setPackagesToScan("com.github.sonus21.rqueue.test.entity"); + factory.setDataSource(dataSource); + return factory; + } + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @AllArgsConstructor + public static class RProcess { + Process process; + RedisNode redisNode; + List out; + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java index 392e550f..875c6bb5 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java @@ -16,54 +16,25 @@ package com.github.sonus21.rqueue.test.application; -import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; import org.springframework.data.redis.listener.RedisMessageListenerContainer; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -import redis.embedded.RedisServer; @Slf4j -public abstract class BaseApplication { - private RedisServer redisServer; - - @Value("${mysql.db.name}") - private String dbName; - - @Value("${spring.redis.port}") - private int redisPort; - - @Value("${spring.redis.host}") - private String redisHost; - - @Value("${use.system.redis:false}") - private boolean useSystemRedis; +public abstract class BaseApplication extends ApplicationBasicConfiguration { @PostConstruct public void postConstruct() { - if (useSystemRedis) { - return; - } - if (redisServer == null) { - redisServer = new RedisServer(redisPort); - redisServer.start(); - } + init(); } @PreDestroy public void preDestroy() { - if (redisServer != null) { - redisServer.stop(); - } + destroy(); } @Bean @@ -71,32 +42,10 @@ public LettuceConnectionFactory redisConnectionFactory() { return new LettuceConnectionFactory(redisHost, redisPort); } - @Bean - public DataSource dataSource() { - EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); - return builder.setType(EmbeddedDatabaseType.H2).setName(dbName).build(); - } - - @Bean - public LocalContainerEntityManagerFactoryBean entityManagerFactory() { - HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(true); - LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); - factory.setJpaVendorAdapter(vendorAdapter); - factory.setPackagesToScan("com.github.sonus21.rqueue.test.entity"); - factory.setDataSource(dataSource()); - return factory; - } - @Bean public RedisMessageListenerContainer container(RedisConnectionFactory redisConnectionFactory) { RedisMessageListenerContainer container = new RedisMessageListenerContainer(); container.setConnectionFactory(redisConnectionFactory); return container; } - - @Bean - public ObjectMapper objectMapper() { - return new ObjectMapper(); - } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/CustomMessageConverterApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/CustomMessageConverterApplication.java new file mode 100644 index 00000000..5ce7f75a --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/CustomMessageConverterApplication.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.application; + +import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; +import java.util.Collections; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import org.springframework.context.annotation.Bean; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.messaging.converter.ByteArrayMessageConverter; + +public abstract class CustomMessageConverterApplication extends ApplicationBasicConfiguration { + @PostConstruct + public void postConstruct() { + init(); + } + + @PreDestroy + public void preDestroy() { + destroy(); + } + + @Bean + public LettuceConnectionFactory redisConnectionFactory() { + return new LettuceConnectionFactory(redisHost, redisPort); + } + + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory( + RedisConnectionFactory redisConnectionFactory) { + SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = + new SimpleRqueueListenerContainerFactory(); + simpleRqueueListenerContainerFactory.setRedisConnectionFactory(redisConnectionFactory); + simpleRqueueListenerContainerFactory.setMessageConverters( + Collections.singletonList(new ByteArrayMessageConverter())); + return simpleRqueueListenerContainerFactory; + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java index 41a7da2c..75a3c6f3 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java @@ -16,17 +16,9 @@ package com.github.sonus21.rqueue.test.application; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -34,104 +26,44 @@ import org.springframework.data.redis.connection.RedisStandaloneConfiguration; import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; import org.springframework.data.redis.listener.RedisMessageListenerContainer; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; import redis.embedded.RedisServer; @Slf4j -public abstract class MultiRedisSprigBaseApplication { - +public abstract class MultiRedisSprigBaseApplication extends ApplicationBasicConfiguration { @Value("${spring.redis2.port}") private int redisPort2; @Value("${spring.redis2.host}") private String redisHost2; - private RedisServer redisServer; - - @Value("${mysql.db.name}") - private String dbName; - - @Value("${spring.redis.port}") - private int redisPort; - - @Value("${spring.redis.host}") - private String redisHost; - - @Value("${use.system.redis:false}") - private boolean useSystemRedis; - private RedisServer redisServer2; - private ExecutorService executor; - private List lines = new ArrayList<>(); - Process process; - @PostConstruct public void postConstruct() { - if (redisServer == null) { - redisServer = new RedisServer(redisPort); - redisServer.start(); - } + init(); if (redisServer2 == null) { redisServer2 = new RedisServer(redisPort2); redisServer2.start(); } - executor = Executors.newSingleThreadExecutor(); - executor.submit( - () -> { - try { - process = Runtime.getRuntime().exec("redis-cli -p " + redisPort + " monitor"); - BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); - String s; - while ((s = br.readLine()) != null) { - lines.add(s); - } - process.waitFor(); - } catch (Exception e) { - log.error("Process call failed", e); - } - }); + monitor(redisHost, redisPort); } @PreDestroy public void preDestroy() { - if (redisServer != null) { - redisServer.stop(); - } + destroy(); if (redisServer2 != null) { redisServer2.stop(); } - if (process != null) { - process.destroy(); - } - for (String line : lines) { - assert line.equals("OK"); + for (RProcess rProcess : processes) { + for (String line : rProcess.out) { + assert line.equals("OK"); + } } } @Bean public LettuceConnectionFactory redisConnectionFactory() { - return new LettuceConnectionFactory(redisHost, redisPort2); - } - - @Bean - public DataSource dataSource() { - EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); - return builder.setType(EmbeddedDatabaseType.H2).setName(dbName).build(); - } - - @Bean - public LocalContainerEntityManagerFactoryBean entityManagerFactory() { - HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(true); - LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); - factory.setJpaVendorAdapter(vendorAdapter); - factory.setPackagesToScan("com.github.sonus21.rqueue.test.entity"); - factory.setDataSource(dataSource()); - return factory; + return new LettuceConnectionFactory(redisHost, redisPort); } @Bean @@ -141,11 +73,6 @@ public RedisMessageListenerContainer container(RedisConnectionFactory redisConne return container; } - @Bean - public ObjectMapper objectMapper() { - return new ObjectMapper(); - } - @Bean public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory() { SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java index c71339b7..6564e45d 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java @@ -22,39 +22,13 @@ import io.lettuce.core.ReadFrom; import java.util.ArrayList; import java.util.List; -import javax.sql.DataSource; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.data.redis.connection.RedisClusterConfiguration; import org.springframework.data.redis.connection.RedisNode; -import org.springframework.data.redis.connection.RedisServer; import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration; import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -public abstract class RedisClusterBaseApplication { - @Value("${mysql.db.name}") - private String dbName; - - @Bean - public DataSource dataSource() { - EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); - return builder.setType(EmbeddedDatabaseType.H2).setName(dbName).build(); - } - - @Bean - public LocalContainerEntityManagerFactoryBean entityManagerFactory() { - HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(true); - LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); - factory.setJpaVendorAdapter(vendorAdapter); - factory.setPackagesToScan("com.github.sonus21.rqueue.test.entity"); - factory.setDataSource(dataSource()); - return factory; - } +public abstract class RedisClusterBaseApplication extends ApplicationBasicConfiguration { @Bean public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory( diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java index da16259a..347ad446 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java @@ -19,8 +19,11 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueWebConfig; -import com.github.sonus21.rqueue.core.QueueRegistry; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueEndpointManager; import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import com.github.sonus21.rqueue.core.RqueueMessageManager; import com.github.sonus21.rqueue.core.RqueueMessageSender; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.support.RqueueMessageFactory; @@ -29,11 +32,15 @@ import com.github.sonus21.rqueue.test.service.ConsumedMessageService; import com.github.sonus21.rqueue.test.service.FailureManager; import com.github.sonus21.rqueue.utils.StringUtils; +import java.time.Duration; +import java.time.Instant; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -44,11 +51,13 @@ public abstract class SpringTestBase extends TestBase { @Autowired protected RqueueMessageTemplate rqueueMessageTemplate; @Autowired protected RqueueConfig rqueueConfig; @Autowired protected RqueueWebConfig rqueueWebConfig; - @Autowired protected RqueueMessageSender messageSender; @Autowired protected RqueueRedisTemplate stringRqueueRedisTemplate; @Autowired protected ConsumedMessageService consumedMessageService; @Autowired protected RqueueMessageListenerContainer rqueueMessageListenerContainer; @Autowired protected FailureManager failureManager; + @Autowired protected RqueueMessageEnqueuer rqueueMessageEnqueuer; + @Autowired protected RqueueEndpointManager rqueueEndpointManager; + @Autowired protected RqueueMessageManager rqueueMessageManager; @Value("${email.queue.name}") protected String emailQueue; @@ -89,8 +98,13 @@ public abstract class SpringTestBase extends TestBase { @Value("${reservation.request.queue.retry.count}") protected int reservationRequestQueueRetryCount; + @Value("${list.email.queue.name}") + protected String listEmailQueue; + protected void enqueue(Object message, String queueName) { - RqueueMessage rqueueMessage = RqueueMessageFactory.buildMessage(message, queueName, null, null); + RqueueMessage rqueueMessage = + RqueueMessageFactory.buildMessage( + rqueueMessageManager.getMessageConverter(), message, queueName, null, null); rqueueMessageTemplate.addMessage(queueName, rqueueMessage); } @@ -98,7 +112,8 @@ protected void enqueue(String queueName, Factory factory, int n) { for (int i = 0; i < n; i++) { Object object = factory.next(i); RqueueMessage rqueueMessage = - RqueueMessageFactory.buildMessage(object, queueName, null, null); + RqueueMessageFactory.buildMessage( + rqueueMessageManager.getMessageConverter(), object, queueName, null, null); rqueueMessageTemplate.addMessage(queueName, rqueueMessage); } } @@ -108,26 +123,29 @@ protected void enqueueIn(String zsetName, Factory factory, Delay delay, int n) { Object object = factory.next(i); long score = delay.getDelay(i); RqueueMessage rqueueMessage = - RqueueMessageFactory.buildMessage(object, zsetName, null, score); + RqueueMessageFactory.buildMessage( + rqueueMessageManager.getMessageConverter(), object, zsetName, null, score); rqueueMessageTemplate.addToZset(zsetName, rqueueMessage, rqueueMessage.getProcessAt()); } } protected void enqueueIn(Object message, String zsetName, long delay) { - RqueueMessage rqueueMessage = RqueueMessageFactory.buildMessage(message, zsetName, null, delay); + RqueueMessage rqueueMessage = + RqueueMessageFactory.buildMessage( + rqueueMessageManager.getMessageConverter(), message, zsetName, null, delay); rqueueMessageTemplate.addToZset(zsetName, rqueueMessage, rqueueMessage.getProcessAt()); } protected Map> getMessageMap(String queueName) { - QueueDetail queueDetail = QueueRegistry.get(queueName); + QueueDetail queueDetail = EndpointRegistry.get(queueName); Map> queueNameToMessage = new HashMap<>(); List messages = rqueueMessageTemplate.readFromList(queueDetail.getQueueName(), 0, -1); queueNameToMessage.put(queueDetail.getQueueName(), messages); - List messagesFromZset = + List messagesInDelayedQueue = rqueueMessageTemplate.readFromZset(queueDetail.getDelayedQueueName(), 0, -1); - queueNameToMessage.put(queueDetail.getDelayedQueueName(), messagesFromZset); + queueNameToMessage.put(queueDetail.getDelayedQueueName(), messagesInDelayedQueue); List messagesInProcessingQueue = rqueueMessageTemplate.readFromZset(queueDetail.getProcessingQueueName(), 0, -1); @@ -149,6 +167,10 @@ protected int getMessageCount(String queueName) { return getMessageCount(Collections.singletonList(queueName)); } + protected int getMessageCount(String queueName, String priority) { + return rqueueMessageManager.getAllMessages(queueName, priority).size(); + } + protected void printQueueStats(List queueNames) { for (String queueName : queueNames) { for (Entry> entry : getMessageMap(queueName).entrySet()) { @@ -160,7 +182,7 @@ protected void printQueueStats(List queueNames) { } protected void cleanQueue(String queue) { - QueueDetail queueDetail = QueueRegistry.get(queue); + QueueDetail queueDetail = EndpointRegistry.get(queue); stringRqueueRedisTemplate.delete(queueDetail.getQueueName()); stringRqueueRedisTemplate.delete(queueDetail.getDelayedQueueName()); stringRqueueRedisTemplate.delete(queueDetail.getProcessingQueueName()); @@ -169,6 +191,152 @@ protected void cleanQueue(String queue) { } } + protected boolean enqueue(String queueName, Object message) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueue(queueName, message); + } else { + return rqueueMessageEnqueuer.enqueue(queueName, message) != null; + } + } + + protected boolean enqueueAt(String queueName, Object message, Date instant) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueAt(queueName, message, instant); + } + return rqueueMessageEnqueuer.enqueueAt(queueName, message, instant) != null; + } + + protected boolean enqueueAt(String queueName, Object message, Instant instant) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueAt(queueName, message, instant); + } + return rqueueMessageEnqueuer.enqueueAt(queueName, message, instant) != null; + } + + protected boolean enqueueAt(String queueName, Object message, long delay) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueAt(queueName, message, delay); + } + return rqueueMessageEnqueuer.enqueueAt(queueName, message, delay) != null; + } + + protected void enqueueIn(String queueName, Object message, long delay) { + if (random.nextBoolean()) { + rqueueMessageSender.enqueueIn(queueName, message, delay); + } else { + rqueueMessageEnqueuer.enqueueIn(queueName, message, delay); + } + } + + protected void enqueueIn(String queueName, Object message, long delay, TimeUnit timeUnit) { + if (random.nextBoolean()) { + rqueueMessageSender.enqueueIn(queueName, message, delay, timeUnit); + } else { + rqueueMessageEnqueuer.enqueueIn(queueName, message, delay, timeUnit); + } + } + + protected boolean enqueueIn(String queueName, Object message, Duration duration) { + if (random.nextBoolean()) { + rqueueMessageSender.enqueueIn(queueName, message, duration); + } + return rqueueMessageEnqueuer.enqueueIn(queueName, message, duration) != null; + } + + protected boolean enqueueWithPriority(String queueName, String priority, Object message) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueWithPriority(queueName, priority, message); + } else { + return rqueueMessageEnqueuer.enqueueWithPriority(queueName, priority, message) != null; + } + } + + protected void enqueueInWithPriority( + String queueName, String priority, Object message, long delay) { + if (random.nextBoolean()) { + rqueueMessageSender.enqueueInWithPriority(queueName, priority, message, delay); + } else { + rqueueMessageEnqueuer.enqueueInWithPriority(queueName, priority, message, delay); + } + } + + protected void enqueueInWithPriority( + String queueName, String priority, Object message, long delay, TimeUnit unit) { + if (random.nextBoolean()) { + rqueueMessageSender.enqueueInWithPriority(queueName, priority, message, delay, unit); + } else { + rqueueMessageEnqueuer.enqueueInWithPriority(queueName, priority, message, delay, unit); + } + } + + protected void enqueueInWithPriority( + String queueName, String priority, Object message, Duration duration) { + if (random.nextBoolean()) { + rqueueMessageSender.enqueueInWithPriority(queueName, priority, message, duration); + } else { + rqueueMessageEnqueuer.enqueueInWithPriority(queueName, priority, message, duration); + } + } + + protected boolean enqueueAtWithPriority( + String queueName, String priority, Object message, Date date) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueAtWithPriority(queueName, priority, message, date); + } else { + return rqueueMessageEnqueuer.enqueueAtWithPriority(queueName, priority, message, date) + != null; + } + } + + protected boolean enqueueAtWithPriority( + String queueName, String priority, Object message, Instant date) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueAtWithPriority(queueName, priority, message, date); + } else { + return rqueueMessageEnqueuer.enqueueAtWithPriority(queueName, priority, message, date) + != null; + } + } + + protected boolean enqueueAtWithPriority( + String queueName, String priority, Object message, long instant) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueAtWithPriority(queueName, priority, message, instant); + } else { + return rqueueMessageEnqueuer.enqueueAtWithPriority(queueName, priority, message, instant) + != null; + } + } + + protected void registerQueue(String queue, String... priorities) { + if (random.nextBoolean()) { + rqueueMessageSender.registerQueue(queue, priorities); + } else { + rqueueEndpointManager.registerQueue(queue, priorities); + } + } + + protected boolean enqueueWithRetry(String queueName, Object message, int retry) { + if (random.nextBoolean()) { + return rqueueMessageSender.enqueueWithRetry(queueName, message, retry); + } + return rqueueMessageEnqueuer.enqueueWithRetry(queueName, message, retry) != null; + } + + protected List getAllMessages(String queueName) { + if (random.nextBoolean()) { + return rqueueMessageSender.getAllMessages(queueName); + } + return rqueueMessageManager.getAllMessages(queueName); + } + + protected boolean deleteAllMessages(String queueName) { + if (random.nextBoolean()) { + return rqueueMessageSender.deleteAllMessages(queueName); + } + return rqueueMessageManager.deleteAllMessages(queueName); + } + public interface Factory { Object next(int i); } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java index e858b443..2b559a7f 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java @@ -17,7 +17,7 @@ package com.github.sonus21.rqueue.test.common; import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; @@ -28,7 +28,7 @@ public abstract class SpringWebTestBase extends SpringTestBase { protected MockMvc mockMvc; protected ObjectMapper mapper = new ObjectMapper(); - @Before + @BeforeEach public void init() { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java index 63062661..de83f4bd 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java @@ -25,13 +25,18 @@ public abstract class TestBase { static { String seed = System.getenv("TEST_SEED"); - long randomSeed; - if (seed == null) { + long randomSeed = 0; + if (seed != null) { + try { + randomSeed = Long.parseLong(seed); + } catch (NumberFormatException e) { + log.error("Invalid number format for log seed {}", seed); + } + } + if (randomSeed == 0) { randomSeed = System.currentTimeMillis(); - } else { - randomSeed = Long.parseLong(seed); } random = new Random(randomSeed); - log.info("Test random seed is {}", randomSeed); + log.error("Test random seed is {}", randomSeed); } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java index 5dd92c1a..05a85eb3 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java @@ -36,30 +36,42 @@ public class ConsumedMessageService { @NonNull private ConsumedMessageRepository consumedMessageRepository; @NonNull private ObjectMapper objectMapper; - public ConsumedMessage save(BaseQueueMessage message, String tag) - throws JsonProcessingException { + public ConsumedMessage save(BaseQueueMessage message, String tag) throws JsonProcessingException { String textMessage = objectMapper.writeValueAsString(message); ConsumedMessage consumedMessage = new ConsumedMessage(message.getId(), textMessage, tag); consumedMessageRepository.save(consumedMessage); return consumedMessage; } + public Collection getConsumedMessages(Collection ids) { + return getMessages(ids).values(); + } + public T getMessage(String id, Class tClass) { return getMessages(Collections.singletonList(id), tClass).get(id); } public Map getMessages(Collection ids, Class tClass) { - Iterable consumedMessages = consumedMessageRepository.findAllById(ids); Map idToMessage = new HashMap<>(); + getMessages(ids) + .values() + .forEach( + consumedMessage -> { + try { + T value = objectMapper.readValue(consumedMessage.getMessage(), tClass); + idToMessage.put(consumedMessage.getId(), value); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + }); + return idToMessage; + } + + public Map getMessages(Collection ids) { + Iterable consumedMessages = consumedMessageRepository.findAllById(ids); + Map idToMessage = new HashMap<>(); consumedMessages.forEach( - consumedMessage -> { - try { - T value = objectMapper.readValue(consumedMessage.getMessage(), tClass); - idToMessage.put(consumedMessage.getId(), value); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - }); + consumedMessage -> idToMessage.put(consumedMessage.getId(), consumedMessage)); return idToMessage; } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/QueueRegistryUpdater.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/QueueRegistryUpdater.java new file mode 100644 index 00000000..91d7a50e --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/QueueRegistryUpdater.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.service; + +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.RqueueEndpointManager; +import com.github.sonus21.rqueue.core.RqueueMessageSender; +import com.github.sonus21.rqueue.models.enums.RqueueMode; +import javax.annotation.PostConstruct; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; + +@Component +@AllArgsConstructor +public class QueueRegistryUpdater { + private final RqueueMessageSender rqueueMessageSender; + private final RqueueEndpointManager rqueueEndpointManager; + private final RqueueConfig rqueueConfig; + + @PostConstruct + public void registerQueues() { + if (!RqueueMode.PRODUCER.equals(rqueueConfig.getMode())) { + return; + } + for (int i = 0; i < 10; i++) { + String queueName = "new_queue_" + i; + String[] priorities = null; + if (i % 3 == 0) { + priorities = new String[2]; + priorities[0] = "high"; + priorities[1] = "low"; + } + if (i % 2 == 0) { + if (priorities != null) { + rqueueMessageSender.registerQueue(queueName, priorities); + } else { + rqueueMessageSender.registerQueue(queueName); + } + } else { + if (priorities != null) { + rqueueEndpointManager.registerQueue(queueName, priorities); + } else { + rqueueEndpointManager.registerQueue(queueName); + } + } + } + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java index ea362da1..b0aad643 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java @@ -31,9 +31,9 @@ public abstract class AllQueueMode extends SpringTestBase { protected void checkGroupConsumer() throws TimedOutException { - rqueueMessageSender.enqueue(chatIndexingQueue, ChatIndexing.newInstance()); - rqueueMessageSender.enqueue(feedGenerationQueue, FeedGeneration.newInstance()); - rqueueMessageSender.enqueue(reservationQueue, Reservation.newInstance()); + enqueue(chatIndexingQueue, ChatIndexing.newInstance()); + enqueue(feedGenerationQueue, FeedGeneration.newInstance()); + enqueue(reservationQueue, Reservation.newInstance()); TimeoutUtils.waitFor( () -> getMessageCount(Arrays.asList(chatIndexingQueue, feedGenerationQueue, reservationQueue)) @@ -43,11 +43,11 @@ protected void checkGroupConsumer() throws TimedOutException { } protected void checkQueueLevelConsumer() throws TimedOutException { - rqueueMessageSender.enqueue(smsQueue, Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "critical", Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "high", Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "medium", Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "low", Sms.newInstance()); + enqueue(smsQueue, Sms.newInstance()); + enqueueWithPriority(smsQueue, "critical", Sms.newInstance()); + enqueueWithPriority(smsQueue, "high", Sms.newInstance()); + enqueueWithPriority(smsQueue, "medium", Sms.newInstance()); + enqueueWithPriority(smsQueue, "low", Sms.newInstance()); TimeoutUtils.waitFor( () -> getMessageCount( @@ -63,11 +63,10 @@ protected void checkQueueLevelConsumer() throws TimedOutException { } protected void testSimpleConsumer() throws TimedOutException { - rqueueMessageSender.enqueue(emailQueue, Email.newInstance()); - rqueueMessageSender.enqueueIn(emailQueue, Email.newInstance(), 1000L); - - rqueueMessageSender.enqueue(jobQueue, Job.newInstance()); - rqueueMessageSender.enqueueIn(jobQueue, Job.newInstance(), 2000L, TimeUnit.MILLISECONDS); + enqueue(emailQueue, Email.newInstance()); + enqueueIn(emailQueue, Email.newInstance(), 1000L); + enqueue(jobQueue, Job.newInstance()); + enqueueIn(jobQueue, Job.newInstance(), 2000L, TimeUnit.MILLISECONDS); TimeoutUtils.waitFor( () -> getMessageCount(Arrays.asList(emailQueue, jobQueue)) == 0, diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageRetryTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/BasicListenerTest.java similarity index 71% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageRetryTest.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/BasicListenerTest.java index 7bb692dd..3ca10e90 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageRetryTest.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/BasicListenerTest.java @@ -17,7 +17,7 @@ package com.github.sonus21.rqueue.test.tests; import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.test.common.SpringTestBase; @@ -26,16 +26,23 @@ import com.github.sonus21.rqueue.test.dto.Notification; import com.github.sonus21.rqueue.test.dto.ReservationRequest; import com.github.sonus21.rqueue.test.entity.ConsumedMessage; +import com.github.sonus21.rqueue.utils.TimeoutUtils; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @Slf4j -public abstract class MessageRetryTest extends SpringTestBase { +public abstract class BasicListenerTest extends SpringTestBase { protected void verifyAfterNRetryTaskIsDeletedFromProcessingQueue() throws TimedOutException { cleanQueue(jobQueue); Job job = Job.newInstance(); failureManager.createFailureDetail(job.getId(), 3, 10); - messageSender.put(jobQueue, job); + rqueueMessageSender.put(jobQueue, job); waitFor( () -> { Job jobInDb = consumedMessageService.getMessage(job.getId(), Job.class); @@ -44,7 +51,7 @@ protected void verifyAfterNRetryTaskIsDeletedFromProcessingQueue() throws TimedO "job to be executed"); waitFor( () -> { - List messages = messageSender.getAllMessages(jobQueue); + List messages = getAllMessages(jobQueue); return !messages.contains(job); }, "message should be deleted from internal storage"); @@ -55,7 +62,7 @@ protected void verifyMessageMovedToDeadLetterQueue() throws TimedOutException { Email email = Email.newInstance(); failureManager.createFailureDetail(email.getId(), -1, 0); log.debug("queue: {} msg: {}", emailQueue, email); - messageSender.put(emailQueue, email, 1000L); + rqueueMessageSender.put(emailQueue, email, 1000L); waitFor( () -> emailRetryCount == failureManager.getFailureCount(email.getId()), 30000000, @@ -71,13 +78,13 @@ protected void verifyMessageIsDiscardedAfterRetries() throws TimedOutException { cleanQueue(notificationQueue); Notification notification = Notification.newInstance(); failureManager.createFailureDetail(notification.getId(), -1, notificationRetryCount); - messageSender.enqueueAt(notificationQueue, notification, System.currentTimeMillis() + 1000L); + enqueueAt(notificationQueue, notification, System.currentTimeMillis() + 1000L); waitFor( () -> notificationRetryCount == failureManager.getFailureCount(notification.getId()), "all retry to be exhausted"); waitFor( () -> { - List messages = messageSender.getAllMessages(notificationQueue); + List messages = getAllMessages(notificationQueue); return !messages.contains(notification); }, "message to be discarded"); @@ -87,7 +94,7 @@ protected void verifyMessageIsDiscardedAfterRetries() throws TimedOutException { public void verifySimpleTaskExecution() throws TimedOutException { cleanQueue(notificationQueue); Notification notification = Notification.newInstance(); - messageSender.enqueue(notificationQueue, notification); + enqueue(notificationQueue, notification); waitFor( () -> { Notification notificationInDb = @@ -101,7 +108,7 @@ public void verifyRetry() throws TimedOutException { cleanQueue(emailQueue); Email email = Email.newInstance(); failureManager.createFailureDetail(email.getId(), emailRetryCount - 1, emailRetryCount - 1); - messageSender.enqueue(emailQueue, email); + enqueue(emailQueue, email); waitFor( () -> failureManager.getFailureCount(email.getId()) == emailRetryCount - 1, "email task needs to be retried"); @@ -117,16 +124,16 @@ public void verifyMessageIsInProcessingQueue() throws TimedOutException { cleanQueue(jobQueue); Job job = Job.newInstance(); failureManager.createFailureDetail(job.getId(), -1, 0); - messageSender.enqueue(jobQueue, job); + enqueue(jobQueue, job); waitFor(() -> failureManager.getFailureCount(job.getId()) >= 3, "Job to be retried"); waitFor( () -> { - List messages = messageSender.getAllMessages(jobQueue); + List messages = getAllMessages(jobQueue); return messages.contains(job); }, "message should be present in internal storage"); // more then one copy should not be present - assertEquals(1, messageSender.getAllMessages(jobQueue).size()); + assertEquals(1, getMessageCount(jobQueue)); } public void verifyMessageIsConsumedByDeadLetterQueueListener() throws TimedOutException { @@ -135,7 +142,7 @@ public void verifyMessageIsConsumedByDeadLetterQueueListener() throws TimedOutEx ReservationRequest request = ReservationRequest.newInstance(); failureManager.createFailureDetail( request.getId(), reservationRequestQueueRetryCount, reservationRequestQueueRetryCount); - messageSender.enqueue(reservationRequestQueue, request); + enqueue(reservationRequestQueue, request); waitFor( () -> failureManager.getFailureCount(request.getId()) >= reservationRequestQueueRetryCount, 60000, @@ -155,4 +162,38 @@ public void verifyMessageIsConsumedByDeadLetterQueueListener() throws TimedOutEx assertEquals(0, getMessageCount(reservationQueue)); assertEquals(0, getMessageCount(reservationRequestDeadLetterQueue)); } + + protected void verifyListMessageListener() throws TimedOutException { + int n = 1 + random.nextInt(10); + List emails = new ArrayList<>(); + List delayedEmails = new ArrayList<>(); + for (int i = 0; i < n; i++) { + emails.add(Email.newInstance()); + delayedEmails.add(Email.newInstance()); + } + enqueue(listEmailQueue, emails); + enqueueIn(listEmailQueue, delayedEmails, 1, TimeUnit.SECONDS); + TimeoutUtils.waitFor( + () -> getMessageCount(listEmailQueue) == 0, "waiting for email list queue to drain"); + Collection messages = + consumedMessageService.getConsumedMessages( + emails.stream().map(Email::getId).collect(Collectors.toList())); + Collection delayedMessages = + consumedMessageService.getConsumedMessages( + delayedEmails.stream().map(Email::getId).collect(Collectors.toList())); + assertEquals(n, messages.size()); + assertEquals(n, delayedEmails.size()); + Set delayedTags = + delayedMessages.stream() + .map(ConsumedMessage::getTag) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + Set simpleTags = + messages.stream() + .map(ConsumedMessage::getTag) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + assertEquals(1, delayedTags.size()); + assertEquals(1, simpleTags.size()); + } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java index e39564c9..0c464697 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java @@ -27,9 +27,9 @@ public abstract class GroupPriorityTest extends SpringTestBase { protected void checkGroupConsumer() throws TimedOutException { - rqueueMessageSender.enqueue(chatIndexingQueue, ChatIndexing.newInstance()); - rqueueMessageSender.enqueue(feedGenerationQueue, FeedGeneration.newInstance()); - rqueueMessageSender.enqueue(reservationQueue, Reservation.newInstance()); + enqueue(chatIndexingQueue, ChatIndexing.newInstance()); + enqueue(feedGenerationQueue, FeedGeneration.newInstance()); + enqueue(reservationQueue, Reservation.newInstance()); TimeoutUtils.waitFor( () -> getMessageCount(Arrays.asList(chatIndexingQueue, feedGenerationQueue, reservationQueue)) diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTests.java similarity index 65% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTest.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTests.java index 581ba2f5..9be5a24c 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTest.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTests.java @@ -17,8 +17,8 @@ package com.github.sonus21.rqueue.test.tests; import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.test.common.SpringTestBase; @@ -32,35 +32,41 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public abstract class MessageChannelTest extends SpringTestBase { - private int messageCount = 200; +public abstract class MessageChannelTests extends SpringTestBase { + private final int messageCount = 200; /** - * This test verified whether any pending message in the delayed queue are moved or not Whenever a - * delayed message is pushed then it's checked whether there're any pending messages on delay - * queue. if expired delayed messages are found on the head then a message is published on delayed - * channel. + * This test verifies whether any pending message in the delayed queue are moved or not whenever a + * delayed message is pushed. During enqueue of delayed message we check whether there are any + * pending messages on the delay queue, if expired delayed messages are found on the head then a + * message is published on delayed channel. */ protected void verifyPublishMessageIsTriggeredOnMessageAddition() throws TimedOutException { String delayedQueueName = rqueueConfig.getDelayedQueueName(emailQueue); enqueueIn(delayedQueueName, i -> Email.newInstance(), i -> -1000L, messageCount); Email email = Email.newInstance(); log.info("adding new message {}", email); - messageSender.enqueueIn(emailQueue, email, Duration.ofMillis(1000)); + enqueueIn(emailQueue, email, Duration.ofMillis(1000)); waitFor( () -> stringRqueueRedisTemplate.getZsetSize(delayedQueueName) <= 1, "one or zero messages in zset"); assertTrue( - "Messages are correctly moved", stringRqueueRedisTemplate.getListSize(rqueueConfig.getQueueName(emailQueue)) - >= messageCount); - assertEquals(messageCount + 1L, messageSender.getAllMessages(emailQueue).size()); + >= messageCount, + "Messages are correctly moved"); + assertEquals(messageCount + 1L, getMessageCount(emailQueue)); } + /** + * This test verifies whether any pending message in the processing queue are moved or not + * whenever a message is pop. During pop of simple message we check whether there are any pending + * messages on the processing queue, if expired messages are found on the head then a message is + * published on processing channel. + */ protected void verifyPublishMessageIsTriggeredOnMessageRemoval() throws TimedOutException { - String processingQueueName = jobQueue; List jobs = new ArrayList<>(); List ids = new ArrayList<>(); int maxDelay = 2000; + String processingQueue = rqueueConfig.getProcessingQueueName(jobQueue); for (int i = 0; i < messageCount; i++) { Job job = Job.newInstance(); jobs.add(job); @@ -69,18 +75,20 @@ protected void verifyPublishMessageIsTriggeredOnMessageRemoval() throws TimedOut if (random.nextBoolean()) { delay = delay * -1; } - enqueueIn(job, processingQueueName, delay); + enqueueIn(job, processingQueue, delay); } TimeoutUtils.sleep(maxDelay); waitFor( - () -> 0 == messageSender.getAllMessages(jobQueue).size(), + () -> 0 == getMessageCount(jobQueue), 30 * Constants.ONE_MILLI, "messages to be consumed"); waitFor( () -> messageCount == consumedMessageService.getMessages(ids, Job.class).size(), + 30 * Constants.ONE_MILLI, "message count to be matched"); waitFor( () -> jobs.containsAll(consumedMessageService.getMessages(ids, Job.class).values()), + 30 * Constants.ONE_MILLI, "All jobs to be executed"); } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java index 4a26dbd8..fa17146d 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java @@ -18,8 +18,6 @@ import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; import static com.google.common.collect.Lists.newArrayList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.test.common.SpringTestBase; @@ -46,23 +44,26 @@ protected void verifyDelayedQueueStatus() throws TimedOutException { if (i < maxMessages / 2) { enqueueIn(notification, rqueueConfig.getDelayedQueueName(notificationQueue), -delay); } else { - messageSender.enqueueAt(notificationQueue, notification, Instant.now().plusMillis(delay)); + enqueueAt(notificationQueue, notification, Instant.now().plusMillis(delay)); } } if (maxDelay == 5000) { Notification notification = Notification.newInstance(); - messageSender.enqueueWithRetry(notificationQueue, notification, 10000); + enqueueWithRetry(notificationQueue, notification, 10000); } - assertTrue( - meterRegistry - .get("delayed.queue.size") - .tag("rqueue", "test") - .tag("queue", notificationQueue) - .gauge() - .value() - > 0); + waitFor( + () -> + meterRegistry + .get("delayed.queue.size") + .tag("rqueue", "test") + .tag("queue", notificationQueue) + .gauge() + .value() + > 0, + 60000, + "stats collection"); waitFor( () -> meterRegistry @@ -72,11 +73,10 @@ protected void verifyDelayedQueueStatus() throws TimedOutException { .gauge() .value() > 0, + 60000, "Message in original queue"); - messageSender.deleteAllMessages(notificationQueue); - waitFor( - () -> messageSender.getAllMessages(notificationQueue).size() == 0, - "notification queue to drain"); + deleteAllMessages(notificationQueue); + waitFor(() -> getMessageCount(notificationQueue) == 0, 60000, "notification queue to drain"); } protected void verifyMetricStatus() throws TimedOutException { @@ -84,17 +84,18 @@ protected void verifyMetricStatus() throws TimedOutException { Job job = Job.newInstance(); failureManager.createFailureDetail(job.getId(), -1, 0); - messageSender.enqueue(jobQueue, job); + enqueue(jobQueue, job); - assertEquals( - 10, - meterRegistry - .get("dead.letter.queue.size") - .tags("rqueue", "test") - .tags("queue", emailQueue) - .gauge() - .value(), - 0); + waitFor( + () -> + meterRegistry + .get("dead.letter.queue.size") + .tags("rqueue", "test") + .tags("queue", emailQueue) + .gauge() + .value() + == 10, + "stats collection"); waitFor( () -> meterRegistry @@ -109,10 +110,10 @@ protected void verifyMetricStatus() throws TimedOutException { } protected void verifyCountStatus() throws TimedOutException { - messageSender.enqueue(emailQueue, Email.newInstance()); + enqueue(emailQueue, Email.newInstance()); Job job = Job.newInstance(); failureManager.createFailureDetail(job.getId(), 1, 1); - messageSender.enqueue(jobQueue, job); + enqueue(jobQueue, job); waitFor( () -> meterRegistry @@ -137,14 +138,15 @@ protected void verifyCountStatus() throws TimedOutException { "message process", () -> printQueueStats(newArrayList(jobQueue, emailQueue, notificationQueue))); - assertEquals( - 0, - meterRegistry - .get("failure.count") - .tags("rqueue", "test") - .tags("queue", emailQueue) - .counter() - .count(), - 0); + waitFor( + () -> + meterRegistry + .get("failure.count") + .tags("rqueue", "test") + .tags("queue", emailQueue) + .counter() + .count() + == 0, + "stats collection"); } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java index 45364185..b8bbd4fc 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java @@ -26,11 +26,11 @@ public abstract class MultiLevelQueueTest extends SpringTestBase { protected void checkQueueLevelConsumer() throws TimedOutException { - rqueueMessageSender.enqueue(smsQueue, Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "critical", Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "high", Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "medium", Sms.newInstance()); - rqueueMessageSender.enqueueWithPriority(smsQueue, "low", Sms.newInstance()); + enqueue(smsQueue, Sms.newInstance()); + enqueueWithPriority(smsQueue, "critical", Sms.newInstance()); + enqueueWithPriority(smsQueue, "high", Sms.newInstance()); + enqueueWithPriority(smsQueue, "medium", Sms.newInstance()); + enqueueWithPriority(smsQueue, "low", Sms.newInstance()); TimeoutUtils.waitFor( () -> getMessageCount( @@ -45,13 +45,11 @@ protected void checkQueueLevelConsumer() throws TimedOutException { } protected void checkQueueLevelConsumerWithDelay() throws TimedOutException { - rqueueMessageSender.enqueue(smsQueue, Sms.newInstance()); - rqueueMessageSender.enqueueInWithPriority(smsQueue, "critical", Sms.newInstance(), 1000L); - rqueueMessageSender.enqueueInWithPriority( - smsQueue, "high", Sms.newInstance(), 1000L, TimeUnit.MILLISECONDS); - rqueueMessageSender.enqueueInWithPriority( - smsQueue, "medium", Sms.newInstance(), Duration.ofMillis(1000L)); - rqueueMessageSender.enqueueInWithPriority(smsQueue, "low", Sms.newInstance(), 1000L); + enqueue(smsQueue, Sms.newInstance()); + enqueueInWithPriority(smsQueue, "critical", Sms.newInstance(), 1000L); + enqueueInWithPriority(smsQueue, "high", Sms.newInstance(), 1000L, TimeUnit.MILLISECONDS); + enqueueInWithPriority(smsQueue, "medium", Sms.newInstance(), Duration.ofMillis(1000L)); + enqueueInWithPriority(smsQueue, "low", Sms.newInstance(), 1000L); TimeoutUtils.waitFor( () -> getMessageCount( diff --git a/rqueue-common-test/src/main/resources/application.properties b/rqueue-common-test/src/main/resources/application.properties index 62fd87c3..e3bf246b 100644 --- a/rqueue-common-test/src/main/resources/application.properties +++ b/rqueue-common-test/src/main/resources/application.properties @@ -39,4 +39,7 @@ reservation.request.queue.retry.count=2 reservation.request.dead.letter.consumer.enabled=false reservation.request.active=false reservation.request.dead.letter.queue.retry.count=1 +list.email.queue.name=email-list-queue +list.email.queue.enabled=false + diff --git a/rqueue-common-test/src/main/resources/logback.xml b/rqueue-common-test/src/main/resources/logback.xml index 1f525748..1b7b7617 100644 --- a/rqueue-common-test/src/main/resources/logback.xml +++ b/rqueue-common-test/src/main/resources/logback.xml @@ -16,10 +16,26 @@ + + + %property{testClass}::%property{testName} [%date{dd-MM-yyyy HH:mm:ss.SSS}] [%thread] %-5level %logger{36} - %msg%n + + log/monitor.log + + log/monitor.%d{yyyy-MM-dd-HH}.log + 30 + 100MB + + + + + + + diff --git a/rqueue-core/build.gradle b/rqueue-core/build.gradle index a04453a5..8fa504f4 100644 --- a/rqueue-core/build.gradle +++ b/rqueue-core/build.gradle @@ -16,6 +16,7 @@ dependencies { compile group: 'org.springframework', name: 'spring-webmvc', version: "${springVersion}" // https://mvnrepository.com/artifact/org.jtwig/jtwig-spring compile group: 'org.jtwig', name: 'jtwig-spring', version: "${twigVersion}" + compile group: 'io.seruco.encoding', name: 'base62', version: "0.1.3" // https://mvnrepository.com/artifact/io.micrometer/micrometer-core compile "io.micrometer:micrometer-core:${microMeterVersion}", optional @@ -23,3 +24,7 @@ dependencies { testCompile "io.lettuce:lettuce-core:${lettuceVersion}" testCompile project(":rqueue-test-util") } + +test { + useJUnitPlatform() +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java index 6e80a019..0c80e6f8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java @@ -16,6 +16,9 @@ package com.github.sonus21.rqueue.config; +import com.github.sonus21.rqueue.models.enums.RqueueMode; +import com.github.sonus21.rqueue.utils.StringUtils; +import java.net.Proxy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -32,28 +35,31 @@ public class RqueueConfig { private final boolean sharedConnection; private final int dbVersion; - @Value("${rqueue.version:2.0.4}") + @Value("${rqueue.version:2.1.0}") private String version; + @Value("${rqueue.latest.version.check.enabled:true}") + private boolean latestVersionCheckEnabled; + @Value("${rqueue.key.prefix:__rq::}") private String prefix; @Value("${rqueue.cluster.mode:true}") private boolean clusterMode; - @Value("${rqueue.simple.queue.prefix:queue::}") + @Value("${rqueue.simple.queue.prefix:}") private String simpleQueuePrefix; - @Value("${rqueue.delayed.queue.prefix:d-queue::}") + @Value("${rqueue.delayed.queue.prefix:}") private String delayedQueuePrefix; - @Value("${rqueue.delayed.queue.channel.prefix:d-channel::}") + @Value("${rqueue.delayed.queue.channel.prefix:}") private String delayedQueueChannelPrefix; - @Value("${rqueue.processing.queue.name.prefix:p-queue::}") + @Value("${rqueue.processing.queue.name.prefix:}") private String processingQueuePrefix; - @Value("${rqueue.processing.queue.channel.prefix:p-channel::}") + @Value("${rqueue.processing.queue.channel.prefix:}") private String processingQueueChannelPrefix; @Value("${rqueue.queues.key.suffix:queues}") @@ -77,43 +83,111 @@ public class RqueueConfig { @Value("${rqueue.default.queue.with.queue.level.priority:-1}") private int defaultQueueWithQueueLevelPriority; + @Value("${rqueue.net.proxy.host:}") + private String proxyHost; + + @Value("${rqueue.net.proxy.port:8000}") + private Integer proxyPort; + + @Value("${rqueue.net.proxy.type:HTTP}") + private Proxy.Type proxyType; + + @Value("${rqueue.message.durability:10080}") + private long messageDurabilityInMinute; + + @Value("${rqueue.message.durability.in-terminal-state:1800}") + private long messageDurabilityInTerminalStateInSecond; + + @Value("${rqueue.system.mode:BOTH}") + private RqueueMode mode; + public String getQueuesKey() { return prefix + queuesKeySuffix; } + private String getSimpleQueueSuffix() { + if (!StringUtils.isEmpty(simpleQueuePrefix)) { + return simpleQueuePrefix; + } + if (dbVersion == 2) { + return "queue::"; + } + return "queue-v2::"; + } + + private String getDelayedQueueSuffix() { + if (!StringUtils.isEmpty(delayedQueuePrefix)) { + return delayedQueuePrefix; + } + if (dbVersion == 2) { + return "d-queue::"; + } + return "d-queue-v2::"; + } + + private String getDelayedQueueChannelSuffix() { + if (!StringUtils.isEmpty(delayedQueueChannelPrefix)) { + return delayedQueueChannelPrefix; + } + if (dbVersion == 2) { + return "d-channel::"; + } + return "d-channel-v2::"; + } + + private String getProcessingQueueSuffix() { + if (!StringUtils.isEmpty(processingQueuePrefix)) { + return processingQueuePrefix; + } + if (dbVersion == 2) { + return "p-queue::"; + } + return "p-queue-v2::"; + } + + private String getProcessingQueueChannelSuffix() { + if (!StringUtils.isEmpty(processingQueueChannelPrefix)) { + return processingQueueChannelPrefix; + } + if (dbVersion == 2) { + return "p-channel::"; + } + return "p-channel-v2::"; + } + public String getQueueName(String queueName) { - if (dbVersion >= 2) { - return prefix + simpleQueuePrefix + getTaggedName(queueName); + if (dbVersion == 1) { + return queueName; } - return queueName; + return prefix + getSimpleQueueSuffix() + getTaggedName(queueName); } public String getDelayedQueueName(String queueName) { - if (dbVersion >= 2) { - return prefix + delayedQueuePrefix + getTaggedName(queueName); + if (dbVersion == 1) { + return "rqueue-delay::" + queueName; } - return "rqueue-delay::" + queueName; + return prefix + getDelayedQueueSuffix() + getTaggedName(queueName); } public String getDelayedQueueChannelName(String queueName) { - if (dbVersion >= 2) { - return prefix + delayedQueueChannelPrefix + getTaggedName(queueName); + if (dbVersion == 1) { + return "rqueue-channel::" + queueName; } - return "rqueue-channel::" + queueName; + return prefix + getDelayedQueueChannelSuffix() + getTaggedName(queueName); } public String getProcessingQueueName(String queueName) { - if (dbVersion >= 2) { - return prefix + processingQueuePrefix + getTaggedName(queueName); + if (dbVersion == 1) { + return "rqueue-processing::" + queueName; } - return "rqueue-processing::" + queueName; + return prefix + getProcessingQueueSuffix() + getTaggedName(queueName); } public String getProcessingQueueChannelName(String queueName) { - if (dbVersion >= 2) { - return prefix + processingQueueChannelPrefix + getTaggedName(queueName); + if (dbVersion == 1) { + return "rqueue-processing-channel::" + queueName; } - return "rqueue-processing-channel::" + queueName; + return prefix + getProcessingQueueChannelSuffix() + getTaggedName(queueName); } public String getLockKey(String key) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java index 115c41a0..21ae1b3e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java @@ -24,8 +24,8 @@ import com.github.sonus21.rqueue.core.DelayedMessageScheduler; import com.github.sonus21.rqueue.core.ProcessingMessageScheduler; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; -import com.github.sonus21.rqueue.core.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; +import com.github.sonus21.rqueue.core.impl.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.utils.RedisUtils; import com.github.sonus21.rqueue.web.view.DateTimeFunction; import com.github.sonus21.rqueue.web.view.DeadLetterQueuesFunction; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java index 52a08bda..bfbda0fe 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java @@ -21,7 +21,7 @@ import com.github.sonus21.rqueue.annotation.RqueueListener; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; -import com.github.sonus21.rqueue.core.RqueueMessageTemplateImpl; +import com.github.sonus21.rqueue.core.impl.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index 04b97247..ecc2f917 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -17,9 +17,11 @@ package com.github.sonus21.rqueue.converter; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.sonus21.rqueue.utils.SerializationUtils; -import java.io.IOException; +import java.util.Collection; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -35,8 +37,9 @@ * vice versa. */ @Slf4j +@SuppressWarnings("unchecked") public class GenericMessageConverter implements MessageConverter { - private static ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper objectMapper = new ObjectMapper(); /** * Convert the payload of a {@link Message} from a serialized form to a typed Object of type @@ -55,15 +58,35 @@ public Object fromMessage(Message message, Class targetClass) { String payload = (String) message.getPayload(); if (SerializationUtils.isJson(payload)) { Msg msg = objectMapper.readValue(payload, Msg.class); - Class c = Thread.currentThread().getContextClassLoader().loadClass(msg.getName()); - return objectMapper.readValue(msg.msg, c); + String[] classNames = splitClassNames(msg.getName()); + if (classNames.length == 1) { + Class c = Thread.currentThread().getContextClassLoader().loadClass(msg.getName()); + return objectMapper.readValue(msg.msg, c); + } + Class envelopeClass = + Thread.currentThread().getContextClassLoader().loadClass(classNames[0]); + Class elementClass = + Thread.currentThread().getContextClassLoader().loadClass(classNames[1]); + JavaType type = + objectMapper + .getTypeFactory() + .constructCollectionType((Class) envelopeClass, elementClass); + return objectMapper.readValue(msg.msg, type); } - } catch (IOException | ClassCastException | ClassNotFoundException e) { - log.warn("Exception", e); + } catch (Exception e) { + log.warn("Deserialization of message {} failed", message, e); } return null; } + private String[] splitClassNames(String name) { + return name.split("#"); + } + + private String getClassName(String name, List payload) { + return name + '#' + payload.get(0).getClass().getName(); + } + /** * Create a {@link Message} whose payload is the result of converting the given payload Object to * serialized form. It ignores all headers components. @@ -77,6 +100,16 @@ public Object fromMessage(Message message, Class targetClass) { @Override public Message toMessage(Object payload, MessageHeaders headers) { String name = payload.getClass().getName(); + if (payload instanceof Collection) { + if (payload instanceof List) { + if (((List) payload).isEmpty()) { + return null; + } + name = getClassName(name, (List) payload); + } else { + return null; + } + } try { String msg = objectMapper.writeValueAsString(payload); Msg message = new Msg(msg, name); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java index c5238bd3..b9a62c1a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java @@ -1,10 +1,12 @@ package com.github.sonus21.rqueue.converter; import com.github.sonus21.rqueue.utils.SerializationUtils; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.data.redis.serializer.SerializationException; +@Slf4j public class RqueueRedisSerializer implements RedisSerializer { private GenericJackson2JsonRedisSerializer jackson2JsonRedisSerializer = new GenericJackson2JsonRedisSerializer(); @@ -22,6 +24,7 @@ public Object deserialize(byte[] bytes) throws SerializationException { try { return jackson2JsonRedisSerializer.deserialize(bytes); } catch (Exception e) { + log.warn("Jackson deserialization has failed {}", new String(bytes), e); return new String(bytes); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java index 0153dbd0..f7c7a66a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java @@ -41,12 +41,12 @@ protected long getNextScheduleTime(String queueName, Long value) { @Override protected String getChannelName(String queueName) { - return QueueRegistry.get(queueName).getDelayedQueueChannelName(); + return EndpointRegistry.get(queueName).getDelayedQueueChannelName(); } @Override protected String getZsetName(String queueName) { - return QueueRegistry.get(queueName).getDelayedQueueName(); + return EndpointRegistry.get(queueName).getDelayedQueueName(); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java similarity index 91% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java index e662feef..08807292 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java @@ -25,11 +25,11 @@ import java.util.function.Function; import java.util.stream.Collectors; -public class QueueRegistry { +public final class EndpointRegistry { private static final Object lock = new Object(); - private static Map queueNameToDetail = new HashMap<>(); + private static final Map queueNameToDetail = new HashMap<>(); - QueueRegistry() {} + private EndpointRegistry() {} public static QueueDetail get(String queueName) { QueueDetail queueDetail = queueNameToDetail.get(queueName); @@ -93,4 +93,8 @@ public static Map getActiveQueueMap() { public static int getActiveQueueCount() { return getActiveQueues().size(); } + + public static int getRegisteredQueueCount() { + return queueNameToDetail.size(); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageConverterFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageConverterFactory.java new file mode 100644 index 00000000..f7e2eddd --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageConverterFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core; + +import com.github.sonus21.rqueue.converter.GenericMessageConverter; +import java.util.ArrayList; +import java.util.List; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.converter.StringMessageConverter; + +public final class MessageConverterFactory { + private MessageConverterFactory() {} + + public static CompositeMessageConverter getMessageConverter( + List messageConverterList) { + List messageConverters = messageConverterList; + if (messageConverterList == null) { + messageConverters = new ArrayList<>(); + } + messageConverters = new ArrayList<>(messageConverters); + // String message converter + StringMessageConverter stringMessageConverter = new StringMessageConverter(); + stringMessageConverter.setSerializedPayloadClass(String.class); + messageConverters.add(stringMessageConverter); + // add generic message converter + messageConverters.add(new GenericMessageConverter()); + return new CompositeMessageConverter(messageConverters); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java index 58c91c6c..e21755d1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java @@ -163,9 +163,10 @@ private void createScheduler(int queueCount) { if (queueCount == 0) { return; } - scheduler = - ThreadUtils.createTaskScheduler( - min(getThreadPoolSize(), queueCount), getThreadNamePrefix(), 60); + int threadPoolSize = min(getThreadPoolSize(), queueCount); + String threadNamePrefix = getThreadNamePrefix(); + int terminationTime = 60; + scheduler = ThreadUtils.createTaskScheduler(threadPoolSize, threadNamePrefix, terminationTime); } private boolean isQueueActive(String queueName) { @@ -195,7 +196,7 @@ protected synchronized void schedule(String queueName, Long startTime, boolean f queueNameToLastMessageSeenTime.put(queueName, currentTime); ScheduledTaskDetail scheduledTaskDetail = queueNameToScheduledTask.get(queueName); - QueueDetail queueDetail = QueueRegistry.get(queueName); + QueueDetail queueDetail = EndpointRegistry.get(queueName); String zsetName = getZsetName(queueName); if (scheduledTaskDetail == null || forceSchedule) { @@ -241,9 +242,9 @@ protected synchronized void schedule(String queueName, Long startTime, boolean f @SuppressWarnings("unchecked") protected void initialize() { - List queueNames = QueueRegistry.getActiveQueues(); + List queueNames = EndpointRegistry.getActiveQueues(); defaultScriptExecutor = new DefaultScriptExecutor<>(redisTemplate); - redisScript = (RedisScript) RedisScriptFactory.getScript(ScriptType.PUSH_MESSAGE); + redisScript = (RedisScript) RedisScriptFactory.getScript(ScriptType.MOVE_EXPIRED_MESSAGE); queueRunningState = new ConcurrentHashMap<>(queueNames.size()); queueNameToScheduledTask = new ConcurrentHashMap<>(queueNames.size()); channelNameToQueueName = new ConcurrentHashMap<>(queueNames.size()); @@ -262,7 +263,7 @@ protected void initialize() { public void onApplicationEvent(RqueueBootstrapEvent event) { doStop(); if (event.isStart()) { - if (QueueRegistry.getActiveQueueCount() == 0) { + if (EndpointRegistry.getActiveQueueCount() == 0) { getLogger().warn("No queues are configured"); return; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java index 56448744..caa369a0 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java @@ -32,7 +32,7 @@ public class ProcessingMessageScheduler extends MessageScheduler { @Override protected void initialize() { super.initialize(); - List queueDetails = QueueRegistry.getActiveQueueDetails(); + List queueDetails = EndpointRegistry.getActiveQueueDetails(); this.queueNameToDelay = new ConcurrentHashMap<>(queueDetails.size()); for (QueueDetail queueDetail : queueDetails) { this.queueNameToDelay.put(queueDetail.getName(), queueDetail.getVisibilityTimeout()); @@ -46,12 +46,12 @@ protected Logger getLogger() { @Override protected String getChannelName(String queueName) { - return QueueRegistry.get(queueName).getProcessingQueueChannelName(); + return EndpointRegistry.get(queueName).getProcessingQueueChannelName(); } @Override protected String getZsetName(String queueName) { - return QueueRegistry.get(queueName).getProcessingQueueName(); + return EndpointRegistry.get(queueName).getProcessingQueueName(); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java index a80ba727..594ee4c9 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java @@ -25,22 +25,22 @@ @SuppressWarnings("unchecked") @ToString -class RedisScriptFactory { - static RedisScript getScript(ScriptType type) { +public class RedisScriptFactory { + public static RedisScript getScript(ScriptType type) { Resource resource = new ClassPathResource(type.getPath()); DefaultRedisScript script = new DefaultRedisScript(); script.setLocation(resource); switch (type) { - case ADD_MESSAGE: + case ENQUEUE_MESSAGE: case MOVE_MESSAGE: - case PUSH_MESSAGE: + case MOVE_EXPIRED_MESSAGE: case MOVE_MESSAGE_LIST_TO_LIST: case MOVE_MESSAGE_LIST_TO_ZSET: case MOVE_MESSAGE_ZSET_TO_ZSET: case MOVE_MESSAGE_ZSET_TO_LIST: script.setResultType(Long.class); return script; - case POP_MESSAGE: + case DEQUEUE_MESSAGE: script.setResultType(RqueueMessage.class); return script; default: @@ -48,16 +48,16 @@ static RedisScript getScript(ScriptType type) { } } - enum ScriptType { - ADD_MESSAGE("scripts/add_message.lua"), - POP_MESSAGE("scripts/pop_message.lua"), + public enum ScriptType { + ENQUEUE_MESSAGE("scripts/enqueue_message.lua"), + DEQUEUE_MESSAGE("scripts/dequeue_message.lua"), MOVE_MESSAGE("scripts/move_message.lua"), - PUSH_MESSAGE("scripts/push_message.lua"), + MOVE_EXPIRED_MESSAGE("scripts/move_expired_message.lua"), MOVE_MESSAGE_LIST_TO_LIST("scripts/move_message_list_to_list.lua"), MOVE_MESSAGE_LIST_TO_ZSET("scripts/move_message_list_to_zset.lua"), MOVE_MESSAGE_ZSET_TO_ZSET("scripts/move_message_zset_to_zset.lua"), MOVE_MESSAGE_ZSET_TO_LIST("scripts/move_message_zset_to_list.lua"); - private String path; + private final String path; ScriptType(String path) { this.path = path; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java new file mode 100644 index 00000000..e4c8ea0d --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core; + +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import java.util.List; + +/** + * Rqueue end point manager, manages the end point related to Rqueue. + * + *

if a queue does not exist then an exception of the {@link + * com.github.sonus21.rqueue.exception.QueueDoesNotExist} will be thrown. In such cases you can + * register a queue using {@link RqueueEndpointManager#registerQueue(String, String...)} + */ +public interface RqueueEndpointManager { + + /** + * Use this method to register any queue, that's only used for sending message. + * + * @param name name of the queue + * @param priorities list of priorities to be used while sending message on this queue. + */ + void registerQueue(String name, String... priorities); + + /** + * Check if a queue is registered. + * + * @param queueName queue that needs to be checked + * @return yes/no + */ + boolean isQueueRegistered(String queueName); + + /** + * Check if a queue is registered. + * + * @param queueName queue that needs to be checked + * @param priority priority of the queue + * @return yes/no + */ + default boolean isQueueRegistered(String queueName, String priority) { + return isQueueRegistered(PriorityUtils.getQueueNameForPriority(queueName, priority)); + } + + /** + * Get queue config for a queue + * + * @param queueName queue name for which configuration has to be fetched + * @return list of queue detail + */ + List getQueueConfig(String queueName); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java index fc84a3d3..53ad8f2b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java @@ -30,22 +30,37 @@ public class RqueueMessage extends SerializableBase implements Cloneable { private static final long serialVersionUID = -3488860960637488519L; + /** The message id, each message has a unique id, generated using */ private String id; + // Queue name on which message was enqueued private String queueName; + // JSON encoded message, this message can be deserialize to actual object with + // the help of MessageUtils#convertMessageToObject method. private String message; + // Any retry count used while enqueueing private Integer retryCount; + // when this message was enqueued, this is in nano second private long queuedTime; + // when this message was supposed to be processed private long processAt; + // The time when it was re-enqueue due to failure. private Long reEnqueuedAt; + // Number of times this message has failed. private int failureCount; public RqueueMessage(String queueName, String message, Integer retryCount, Long delay) { this.queueName = queueName; this.message = message; this.retryCount = retryCount; - this.queuedTime = System.currentTimeMillis(); this.id = UUID.randomUUID().toString(); - this.processAt = this.queuedTime; + initTime(delay); + } + + private void initTime(Long delay) { + // Monotonic increasing queued time + // This is used to check duplicate message in executor + this.queuedTime = System.nanoTime(); + this.processAt = System.currentTimeMillis(); if (delay != null) { this.processAt += delay; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java new file mode 100644 index 00000000..3782f07c --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java @@ -0,0 +1,556 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +/** + * RqueueMessageEnqueuer enqueue message to Redis queue using different mechanism. Use any of the + * methods from this interface to enqueue message to any queue. Queue must exist, if a queue does + * not exist then it will throw an error of the {@link + * com.github.sonus21.rqueue.exception.QueueDoesNotExist}. In such case register your queue using + * {@link RqueueEndpointManager#registerQueue(String, String...)} method. + * + *

There are three types of interfaces in this 1. enqueueXYZ 2. enqueueInXYZ 3. enqueueAtXYZ + * + *

Messages enqueue using enqueueXYZ shall be consumed as soon as possible + * + *

Messages enqueue using enqueueInXYZ shall be consumed once the given time is elapsed, like in + * 30 seconds. + * + *

Messages send using enqueueAtXYZ shall be consumed as soon as the given time is reached for + * example 3PM tomorrow. + * + * @author Sonu Kumar + */ +public interface RqueueMessageEnqueuer { + /** + * Enqueue a message on given queue without any delay, consume as soon as possible. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @return message id on successful enqueue otherwise null. + */ + String enqueue(String queueName, Object message); + + /** + * Enqueue a message on given queue without any delay, consume as soon as possible. + * + * @param queueName on which queue message has to be send + * @param messageId message id + * @param message message object it could be any arbitrary object. + * @return message was enqueue successfully or failed. + */ + boolean enqueue(String queueName, String messageId, Object message); + + /** + * Enqueue a message on the given queue with the given retry count. This message would not be + * consumed more than the specified time due to failure in underlying systems. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or send + * to dead letter queue configured using {@link RqueueListener#numRetries()} + * @return message id on successful enqueue otherwise null. + */ + String enqueueWithRetry(String queueName, Object message, int retryCount); + + /** + * Enqueue a message on the given queue with the given retry count. This message would not be + * consumed more than the specified time due to failure in underlying systems. + * + * @param queueName on which queue message has to be send + * @param messageId message id for this message. + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or send + * to dead letter queue configured using {@link RqueueListener#numRetries()} + * @return message was enqueue successfully or failed. + */ + boolean enqueueWithRetry(String queueName, String messageId, Object message, int retryCount); + + /** + * Enqueue a message on given queue, that will be consumed as soon as possible. + * + * @param queueName on which queue message has to be send + * @param priority the priority for this message, like high, low, medium etc + * @param message message object it could be any arbitrary object. + * @return message id on successful enqueue otherwise null. + */ + String enqueueWithPriority(String queueName, String priority, Object message); + + /** + * Enqueue a message on given queue, that will be consumed as soon as possible. + * + * @param queueName on which queue message has to be send + * @param priority the priority for this message, like high, low, medium etc + * @param messageId the message id for this message + * @param message message object it could be any arbitrary object. + * @return message was enqueued successfully or not. + */ + boolean enqueueWithPriority(String queueName, String priority, String messageId, Object message); + + /** + * Enqueue unique message on a given queue without any delay, consume as soon as possible. + * + * @param queueName on which queue message has to be send + * @param messageId the message id for uniqueness + * @param message message object it could be any arbitrary object. + * @return message id on successful enqueue otherwise null. + */ + default boolean enqueueUnique(String queueName, String messageId, Object message) { + return enqueue(queueName, messageId, message); + } + + /** + * Enqueue unique message on given queue, that will be consumed as soon as possible. + * + * @param queueName on which queue message has to be send + * @param priority the priority for this message, like high, low, medium etc + * @param messageId the message id for this message + * @param message message object it could be any arbitrary object. + * @return message was enqueue successfully or failed. + */ + default boolean enqueueUniqueWithPriority( + String queueName, String priority, String messageId, Object message) { + return enqueueWithPriority(queueName, priority, messageId, message); + } + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse, for example process in 10 seconds + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milli seconds + * @return message id on successful enqueue otherwise null. + */ + String enqueueIn(String queueName, Object message, long delayInMilliSecs); + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse. + * + * @param queueName on which queue message has to be send + * @param messageId the message id, using which this message will be identified + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milli seconds + * @return message was enqueue successfully or failed. + */ + boolean enqueueIn(String queueName, String messageId, Object message, long delayInMilliSecs); + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be executed. + * @return message id on successful enqueue otherwise null. + */ + default String enqueueIn(String queueName, Object message, Duration delay) { + return enqueueIn(queueName, message, delay.toMillis()); + } + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse. + * + * @param queueName on which queue message has to be send + * @param messageId the message id, using which this message will be identified + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be executed. + * @return success or failure. + */ + default boolean enqueueIn(String queueName, String messageId, Object message, Duration delay) { + return enqueueIn(queueName, messageId, message, delay.toMillis()); + } + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the specified delay elapse. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be executed. + * @param unit unit of the delay + * @return message id on successful enqueue otherwise null. + */ + default String enqueueIn(String queueName, Object message, long delay, TimeUnit unit) { + return enqueueIn(queueName, message, unit.toMillis(delay)); + } + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the specified delay elapse. + * + * @param queueName on which queue message has to be send + * @param messageId message id using which this message can be identified + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be executed. + * @param unit unit of the delay + * @return success or failure. + */ + default boolean enqueueIn( + String queueName, String messageId, Object message, long delay, TimeUnit unit) { + return enqueueIn(queueName, messageId, message, unit.toMillis(delay)); + } + + /** + * Enqueue a task that would be scheduled to run in the specified milli seconds. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or sent + * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} + * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * listener when number of millisecond has elapsed. + * @return message id on successful enqueue otherwise {@literal null} + */ + String enqueueInWithRetry( + String queueName, Object message, int retryCount, long delayInMilliSecs); + + /** + * Enqueue a task that would be scheduled to run in the specified milli seconds. + * + * @param queueName on which queue message has to be send + * @param messageId the message identifier + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or sent + * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} + * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * listener when number of millisecond has elapsed. + * @return message was enqueue successfully or failed. + */ + boolean enqueueInWithRetry( + String queueName, String messageId, Object message, int retryCount, long delayInMilliSecs); + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milli seconds + * @return message id on successful enqueue otherwise {@literal null}. + */ + String enqueueInWithPriority( + String queueName, String priority, Object message, long delayInMilliSecs); + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milli seconds + * @return message was enqueue successfully or failed. + */ + boolean enqueueInWithPriority( + String queueName, String priority, String messageId, Object message, long delayInMilliSecs); + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be consumed. + * @return message id on successful enqueue otherwise {@literal null}. + */ + default String enqueueInWithPriority( + String queueName, String priority, Object message, Duration delay) { + return enqueueInWithPriority(queueName, priority, message, delay.toMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be consumed. + * @return message was enqueue successfully or failed. + */ + default boolean enqueueInWithPriority( + String queueName, String priority, String messageId, Object message, Duration delay) { + return enqueueInWithPriority(queueName, priority, messageId, message, delay.toMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be consumed. + * @param unit unit of the delay + * @return message id on a successful enqueue otherwise {@literal null}. + */ + default String enqueueInWithPriority( + String queueName, String priority, Object message, long delay, TimeUnit unit) { + return enqueueInWithPriority(queueName, priority, message, unit.toMillis(delay)); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be consumed. + * @param unit unit of the delay + * @return message was enqueue successfully or failed. + */ + default boolean enqueueInWithPriority( + String queueName, + String priority, + String messageId, + Object message, + long delay, + TimeUnit unit) { + return enqueueInWithPriority(queueName, priority, messageId, message, unit.toMillis(delay)); + } + + /** + * Enqueue a message on given queue with delay, consume as soon as the delayed is expired. + * + * @param queueName on which queue message has to be send + * @param messageId the message id for uniqueness + * @param message message object it could be any arbitrary object. + * @param delayInMillisecond total execution delay + * @return message id on successful enqueue otherwise {@literal null}. + */ + default boolean enqueueUniqueIn( + String queueName, String messageId, Object message, long delayInMillisecond) { + return enqueueIn(queueName, messageId, message, delayInMillisecond); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param startTimeInMilliSeconds time at which this message has to be consumed. + * @return message id on successful enqueue otherwise {@literal null}. + */ + default String enqueueAt(String queueName, Object message, long startTimeInMilliSeconds) { + return enqueueIn(queueName, message, startTimeInMilliSeconds - System.currentTimeMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param messageId message id + * @param message message object it could be any arbitrary object. + * @param startTimeInMilliSeconds time at which this message has to be consumed. + * @return message was enqueued successfully or failed. + */ + default boolean enqueueAt( + String queueName, String messageId, Object message, long startTimeInMilliSeconds) { + return enqueueIn( + queueName, messageId, message, startTimeInMilliSeconds - System.currentTimeMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param starTime time at which this message has to be consumed. + * @return message id on successful enqueue otherwise {@literal null}. + */ + default String enqueueAt(String queueName, Object message, Instant starTime) { + return enqueueAt(queueName, message, starTime.toEpochMilli()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param starTime time at which this message has to be consumed. + * @return message was enqueued successfully or failed. + */ + default boolean enqueueAt(String queueName, String messageId, Object message, Instant starTime) { + return enqueueAt(queueName, messageId, message, starTime.toEpochMilli()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param starTime time at which this message has to be consumed. + * @return message id on successful enqueue otherwise {@literal null}. + */ + default String enqueueAt(String queueName, Object message, Date starTime) { + return enqueueAt(queueName, message, starTime.toInstant()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param starTime time at which this message has to be consumed. + * @return message was enqueued successfully or failed. + */ + default boolean enqueueAt(String queueName, String messageId, Object message, Date starTime) { + return enqueueAt(queueName, messageId, message, starTime.toInstant()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param startTimeInMilliSecond time at which the message would be consumed. + * @return message id on successful enqueue otherwise {@literal null}. + */ + default String enqueueAtWithPriority( + String queueName, String priority, Object message, long startTimeInMilliSecond) { + return enqueueInWithPriority( + queueName, priority, message, startTimeInMilliSecond - System.currentTimeMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param startTimeInMilliSecond time at which the message would be consumed. + * @return message was enqueue successfully or failed. + */ + default boolean enqueueAtWithPriority( + String queueName, + String priority, + String messageId, + Object message, + long startTimeInMilliSecond) { + return enqueueInWithPriority( + queueName, + priority, + messageId, + message, + startTimeInMilliSecond - System.currentTimeMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param startTime time at which message is supposed to consume + * @return message id on successful enqueue otherwise {@literal null} + */ + default String enqueueAtWithPriority( + String queueName, String priority, Object message, Instant startTime) { + return enqueueAtWithPriority(queueName, priority, message, startTime.toEpochMilli()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param messageId the message id + * @param instant time at which message is supposed to consume + * @return message was enqueue successfully or failed. + */ + default boolean enqueueAtWithPriority( + String queueName, String priority, String messageId, Object message, Instant instant) { + return enqueueAtWithPriority(queueName, priority, messageId, message, instant.toEpochMilli()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param time time at which message would be consumed. + * @return a message id on successful enqueue otherwise {@literal null} + */ + default String enqueueAtWithPriority( + String queueName, String priority, Object message, Date time) { + return enqueueAtWithPriority(queueName, priority, message, time.toInstant()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param time time at which message would be consumed. + * @return message was enqueue successfully or failed. + */ + default boolean enqueueAtWithPriority( + String queueName, String priority, String messageId, Object message, Date time) { + return enqueueAtWithPriority(queueName, priority, messageId, message, time.toInstant()); + } + + /** + * Schedule unique messages on the given queue at the provided time. It will be available to + * consume as soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param messageId a unique identifier message id for this message + * @param timeInMilliSeconds time at which this message has to be consumed. + * @return message was enqueue successfully or failed. + */ + default boolean enqueueUniqueAt( + String queueName, String messageId, Object message, long timeInMilliSeconds) { + return enqueueUniqueIn( + queueName, messageId, message, timeInMilliSeconds - System.currentTimeMillis()); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageManager.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageManager.java new file mode 100644 index 00000000..ea7bd004 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageManager.java @@ -0,0 +1,148 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core; + +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import java.util.List; +import org.springframework.messaging.converter.MessageConverter; + +/** + * Rqueue Message Manager manages messages related to a queue. + * + *

One or more messages can be deleted from a queue, not only this we can delete entire queue, + * that will delete messages related to a queue. + * + *

We can also check whether the given message is enqueued or not. + */ +public interface RqueueMessageManager { + /** + * Very dangerous method it will delete all messages in a queue + * + * @param queueName queue name + * @return fail/success + */ + boolean deleteAllMessages(String queueName); + + /** + * Delete all message for the given that has some priority like high,medium and low + * + * @param queueName queue name + * @param priority the priority for the queue + * @return fail/success + */ + default boolean deleteAllMessages(String queueName, String priority) { + return deleteAllMessages(PriorityUtils.getQueueNameForPriority(queueName, priority)); + } + + /** + * Find all messages stored on a given queue, it considers three types of messages + * + *

1. In-Progress/In-Flight messages 2. Scheduled messages 3. Waiting for execution + * + * @param queueName queue name to be query for + * @return list of messages + */ + List getAllMessages(String queueName); + + /** + * Find all messages stored on a given queue with given priority, this method is extension to the + * method {@link #getAllMessages(String)} + * + * @param queueName queue name to be query for + * @param priority the priority of the queue + * @return list of enqueued messages. + */ + default List getAllMessages(String queueName, String priority) { + return getAllMessages(PriorityUtils.getQueueNameForPriority(queueName, priority)); + } + + /** + * Find the enqueued message. It will provide message upto {@link + * RqueueConfig#messageDurabilityInTerminalStateInSecond} post consumption. Post that messages are + * deleted automatically. + * + * @param queueName queue name on which message was enqueued + * @param id message id + * @return the enqueued message, it could be null if message is not found or it's deleted. + */ + Object getMessage(String queueName, String id); + + /** + * Extension to the method {@link #getMessage(String, String)}, this provides the message for the + * priority queue. + * + * @param queueName queue name on which message was enqueued + * @param priority the priority of the queue + * @param id message id + * @return the enqueued message, it could be null if message is not found or it's deleted. + */ + default Object getMessage(String queueName, String priority, String id) { + return getMessage(PriorityUtils.getQueueNameForPriority(queueName, priority), id); + } + + /** + * Extension to method {@link #getMessage(String, String)}, instead of providing message it + * returns true/false. + * + * @param queueName queue name on which message was enqueued + * @param id message id + * @return whether the message exist or not + */ + boolean exist(String queueName, String id); + + /** + * Extension to the method {@link #exist(String, String)}, that checks message for priority queue. + * + * @param queueName queue name on which message was enqueued + * @param priority priority of the given queue + * @param id message id + * @return whether the message exist or not + */ + default boolean exist(String queueName, String priority, String id) { + return exist(PriorityUtils.getQueueNameForPriority(queueName, priority), id); + } + + /** + * Extension to the method {@link #getMessage(String, String)}, this returns internal message. + * + * @param queueName queue name on which message was enqueued + * @param id message id + * @return the enqueued message + */ + RqueueMessage getRqueueMessage(String queueName, String id); + + /** + * Extension to the method {@link #getRqueueMessage(String, String)} + * + * @param queueName queue name on which message was enqueued + * @param priority the priority of the queue + * @param id message id + * @return the enqueued message + */ + default RqueueMessage getRqueueMessage(String queueName, String priority, String id) { + return getRqueueMessage(PriorityUtils.getQueueNameForPriority(queueName, priority), id); + } + + boolean deleteMessage(String queueName, String id); + + default boolean deleteMessage(String queueName, String priority, String id) { + return deleteMessage(PriorityUtils.getQueueNameForPriority(queueName, priority), id); + } + + MessageConverter getMessageConverter(); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java index f1e84e45..98ae47b9 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java @@ -40,8 +40,12 @@ *

Messages send using enqueueAtXYZ shall be consumed as soon as the given time is reached for * example 3PM tomorrow. * + * @see RqueueMessageEnqueuer + * @see RqueueMessageManager + * @see RqueueEndpointManager * @author Sonu Kumar */ +@Deprecated public interface RqueueMessageSender { /** * Enqueue a message on given queue without any delay, consume as soon as possible. diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java index 587bd859..a21fd47d 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java @@ -62,4 +62,6 @@ MessageMoveResult moveMessageZsetToZset( Long removeElementFromZset(String zsetName, RqueueMessage rqueueMessage); List> readFromZsetWithScore(String name, long start, long end); + + Long getScore(String delayedQueueName, RqueueMessage rqueueMessage); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueRedisListenerContainerFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueRedisListenerContainerFactory.java index 21f1528b..ab07e8b3 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueRedisListenerContainerFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueRedisListenerContainerFactory.java @@ -40,21 +40,21 @@ public class RqueueRedisListenerContainerFactory @Override public void destroy() throws Exception { - if (isMyContainer()) { + if (notSharedContainer()) { container.destroy(); } } @Override public void start() { - if (isMyContainer()) { + if (notSharedContainer()) { container.start(); } } @Override public void stop() { - if (isMyContainer()) { + if (notSharedContainer()) { container.stop(); } } @@ -71,15 +71,25 @@ public RedisMessageListenerContainer getContainer() { return this.container; } - private boolean isMyContainer() { + private boolean notSharedContainer() { return container != null && !sharedContainer; } + private void createContainer() { + container = new RedisMessageListenerContainer(); + container.setConnectionFactory(rqueueConfig.getConnectionFactory()); + container.afterPropertiesSet(); + } + @Override public void afterPropertiesSet() throws Exception { if (!rqueueSchedulerConfig.isRedisEnabled()) { return; } + if (!rqueueConfig.isSharedConnection()) { + createContainer(); + return; + } if (rqueueConfig.isSharedConnection() || rqueueSchedulerConfig.isListenerShared()) { if (systemContainer != null) { container = systemContainer; @@ -87,8 +97,6 @@ public void afterPropertiesSet() throws Exception { return; } } - container = new RedisMessageListenerContainer(); - container.setConnectionFactory(rqueueConfig.getConnectionFactory()); - container.afterPropertiesSet(); + createContainer(); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java new file mode 100644 index 00000000..5ee32390 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core.impl; + +import static com.github.sonus21.rqueue.utils.Constants.MIN_DELAY; +import static com.github.sonus21.rqueue.utils.MessageUtils.buildMessage; +import static com.github.sonus21.rqueue.utils.Validator.validateQueue; +import static org.springframework.util.Assert.notNull; + +import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.MessageConverterFactory; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.models.db.MessageStatus; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.MessageConverter; + +@Slf4j +@SuppressWarnings("WeakerAccess") +abstract class BaseMessageSender { + protected CompositeMessageConverter messageConverter; + protected RqueueMessageTemplate messageTemplate; + @Autowired protected RqueueRedisTemplate stringRqueueRedisTemplate; + @Autowired protected RqueueConfig rqueueConfig; + @Autowired protected RqueueMessageMetadataService rqueueMessageMetadataService; + + BaseMessageSender(RqueueMessageTemplate messageTemplate) { + this(messageTemplate, Collections.emptyList()); + } + + BaseMessageSender( + RqueueMessageTemplate messageTemplate, List messageConverters) { + notNull(messageTemplate, "messageTemplate cannot be null"); + notNull(messageConverters, "messageConverters cannot be null"); + this.messageTemplate = messageTemplate; + this.messageConverter = MessageConverterFactory.getMessageConverter(messageConverters); + } + + private void storeMessageMetadata(RqueueMessage rqueueMessage, Long delayInMillis) { + MessageMetadata messageMetadata = new MessageMetadata(rqueueMessage, MessageStatus.ENQUEUED); + Duration duration; + if (delayInMillis != null) { + duration = Duration.ofMillis(2 * delayInMillis); + long minutes = duration.toMinutes(); + if (minutes < rqueueConfig.getMessageDurabilityInMinute()) { + duration = Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute()); + } + } else { + duration = Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute()); + } + rqueueMessageMetadataService.save(messageMetadata, duration); + } + + private RqueueMessage constructMessage( + String queueName, + String messageId, + Object message, + Integer retryCount, + Long delayInMilliSecs) { + RqueueMessage rqueueMessage = + buildMessage(messageConverter, queueName, message, retryCount, delayInMilliSecs); + if (messageId != null) { + rqueueMessage.setId(messageId); + } + return rqueueMessage; + } + + private void enqueue( + QueueDetail queueDetail, RqueueMessage rqueueMessage, Long delayInMilliSecs) { + if (delayInMilliSecs == null || delayInMilliSecs <= MIN_DELAY) { + messageTemplate.addMessage(queueDetail.getQueueName(), rqueueMessage); + } else { + messageTemplate.addMessageWithDelay( + queueDetail.getDelayedQueueName(), + queueDetail.getDelayedQueueChannelName(), + rqueueMessage); + } + } + + protected String pushMessage( + String queueName, + String messageId, + Object message, + Integer retryCount, + Long delayInMilliSecs) { + QueueDetail queueDetail = EndpointRegistry.get(queueName); + RqueueMessage rqueueMessage = + constructMessage(queueName, messageId, message, retryCount, delayInMilliSecs); + try { + enqueue(queueDetail, rqueueMessage, delayInMilliSecs); + storeMessageMetadata(rqueueMessage, delayInMilliSecs); + } catch (Exception e) { + log.error("Queue: {} Message {} could not be pushed {}", queueName, rqueueMessage, e); + return null; + } + return rqueueMessage.getId(); + } + + protected void registerQueueInternal(String queueName, String... priorities) { + validateQueue(queueName); + notNull(priorities, "priorities cannot be null"); + QueueDetail queueDetail = + QueueDetail.builder() + .name(queueName) + .active(false) + .queueName(rqueueConfig.getQueueName(queueName)) + .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName)) + .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName)) + .processingQueueName(rqueueConfig.getProcessingQueueName(queueName)) + .processingQueueChannelName(rqueueConfig.getProcessingQueueChannelName(queueName)) + .build(); + EndpointRegistry.register(queueDetail); + for (String priority : priorities) { + String suffix = PriorityUtils.getSuffix(priority); + queueDetail = + QueueDetail.builder() + .name(queueName + suffix) + .active(false) + .queueName(rqueueConfig.getQueueName(queueName) + suffix) + .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName) + suffix) + .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName) + suffix) + .processingQueueName(rqueueConfig.getProcessingQueueName(queueName) + suffix) + .processingQueueChannelName( + rqueueConfig.getProcessingQueueChannelName(queueName) + suffix) + .build(); + EndpointRegistry.register(queueDetail); + } + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java new file mode 100644 index 00000000..141c5792 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core.impl; + +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueEndpointManager; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.exception.QueueDoesNotExist; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.utils.Constants; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.util.CollectionUtils; + +public class RqueueEndpointManagerImpl extends BaseMessageSender implements RqueueEndpointManager { + + public RqueueEndpointManagerImpl(RqueueMessageTemplate messageTemplate) { + super(messageTemplate); + } + + @Override + public void registerQueue(String name, String... priorities) { + registerQueueInternal(name, priorities); + } + + @Override + public boolean isQueueRegistered(String queueName) { + try { + EndpointRegistry.get(queueName); + return true; + } catch (QueueDoesNotExist e) { + return false; + } + } + + @Override + public List getQueueConfig(String queueName) { + QueueDetail queueDetail = EndpointRegistry.get(queueName); + Map priorityMap = queueDetail.getPriority(); + if (CollectionUtils.isEmpty(priorityMap)) { + return Collections.singletonList(queueDetail); + } + Map localPriorityMap = new HashMap<>(priorityMap); + localPriorityMap.remove(Constants.DEFAULT_PRIORITY_KEY); + List queueDetails = new ArrayList<>(); + queueDetails.add(queueDetail); + for (String priority : localPriorityMap.keySet()) { + queueDetails.add( + EndpointRegistry.get(PriorityUtils.getQueueNameForPriority(queueName, priority))); + } + return queueDetails; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java new file mode 100644 index 00000000..9cf531f1 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java @@ -0,0 +1,173 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core.impl; + +import static com.github.sonus21.rqueue.utils.Validator.validateDelay; +import static com.github.sonus21.rqueue.utils.Validator.validateMessage; +import static com.github.sonus21.rqueue.utils.Validator.validateMessageId; +import static com.github.sonus21.rqueue.utils.Validator.validatePriority; +import static com.github.sonus21.rqueue.utils.Validator.validateQueue; +import static com.github.sonus21.rqueue.utils.Validator.validateRetryCount; + +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.springframework.messaging.converter.MessageConverter; + +@Slf4j +public class RqueueMessageEnqueuerImpl extends BaseMessageSender implements RqueueMessageEnqueuer { + + public RqueueMessageEnqueuerImpl(RqueueMessageTemplate messageTemplate) { + super(messageTemplate, Collections.emptyList()); + } + + public RqueueMessageEnqueuerImpl( + RqueueMessageTemplate messageTemplate, List messageConverters) { + super(messageTemplate, messageConverters); + } + + @Override + public String enqueue(String queueName, Object message) { + validateQueue(queueName); + validateMessage(message); + return pushMessage(queueName, null, message, null, null); + } + + @Override + public boolean enqueue(String queueName, String messageId, Object message) { + validateQueue(queueName); + validateMessageId(messageId); + validateMessage(message); + return pushMessage(queueName, messageId, message, null, null) != null; + } + + @Override + public String enqueueWithRetry(String queueName, Object message, int retryCount) { + validateQueue(queueName); + validateMessage(message); + validateRetryCount(retryCount); + return pushMessage(queueName, null, message, retryCount, null); + } + + @Override + public boolean enqueueWithRetry( + String queueName, String messageId, Object message, int retryCount) { + validateQueue(queueName); + validateMessageId(messageId); + validateMessage(message); + validateRetryCount(retryCount); + return pushMessage(queueName, messageId, message, retryCount, null) != null; + } + + @Override + public String enqueueWithPriority(String queueName, String priority, Object message) { + validateQueue(queueName); + validatePriority(priority); + validateMessage(message); + return pushMessage( + PriorityUtils.getQueueNameForPriority(queueName, priority), null, message, null, null); + } + + @Override + public boolean enqueueWithPriority( + String queueName, String priority, String messageId, Object message) { + validateQueue(queueName); + validatePriority(priority); + validateMessageId(messageId); + validateMessage(message); + return pushMessage( + PriorityUtils.getQueueNameForPriority(queueName, priority), + messageId, + message, + null, + null) + != null; + } + + @Override + public String enqueueIn(String queueName, Object message, long delayInMilliSecs) { + validateQueue(queueName); + validateMessage(message); + validateDelay(delayInMilliSecs); + return pushMessage(queueName, null, message, null, delayInMilliSecs); + } + + @Override + public boolean enqueueIn( + String queueName, String messageId, Object message, long delayInMilliSecs) { + validateQueue(queueName); + validateMessageId(messageId); + validateMessage(message); + validateDelay(delayInMilliSecs); + return pushMessage(queueName, messageId, message, null, delayInMilliSecs) != null; + } + + @Override + public String enqueueInWithRetry( + String queueName, Object message, int retryCount, long delayInMilliSecs) { + validateQueue(queueName); + validateMessage(message); + validateRetryCount(retryCount); + validateDelay(delayInMilliSecs); + return pushMessage(queueName, null, message, retryCount, delayInMilliSecs); + } + + @Override + public boolean enqueueInWithRetry( + String queueName, String messageId, Object message, int retryCount, long delayInMilliSecs) { + validateQueue(queueName); + validateMessageId(messageId); + validateMessage(message); + validateRetryCount(retryCount); + validateDelay(delayInMilliSecs); + return pushMessage(queueName, messageId, message, retryCount, delayInMilliSecs) != null; + } + + @Override + public String enqueueInWithPriority( + String queueName, String priority, Object message, long delayInMilliSecs) { + validateQueue(queueName); + validatePriority(priority); + validateMessage(message); + validateDelay(delayInMilliSecs); + return pushMessage( + PriorityUtils.getQueueNameForPriority(queueName, priority), + null, + message, + null, + delayInMilliSecs); + } + + @Override + public boolean enqueueInWithPriority( + String queueName, String priority, String messageId, Object message, long delayInMilliSecs) { + validateQueue(queueName); + validatePriority(priority); + validateMessage(message); + validateDelay(delayInMilliSecs); + return pushMessage( + PriorityUtils.getQueueNameForPriority(queueName, priority), + messageId, + message, + null, + delayInMilliSecs) + != null; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java new file mode 100644 index 00000000..3b11cc93 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java @@ -0,0 +1,126 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core.impl; + +import com.github.sonus21.rqueue.common.RqueueLockManager; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageManager; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.exception.LockCanNotBeAcquired; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.utils.MessageUtils; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.support.MessageBuilder; + +@Slf4j +public class RqueueMessageManagerImpl extends BaseMessageSender implements RqueueMessageManager { + @Autowired private RqueueLockManager rqueueLockManager; + + public RqueueMessageManagerImpl(RqueueMessageTemplate messageTemplate) { + super(messageTemplate, Collections.emptyList()); + } + + public RqueueMessageManagerImpl( + RqueueMessageTemplate messageTemplate, List messageConverters) { + super(messageTemplate, messageConverters); + } + + @Override + public boolean deleteAllMessages(String queueName) { + QueueDetail queueDetail = EndpointRegistry.get(queueName); + try { + stringRqueueRedisTemplate.delete(queueDetail.getQueueName()); + stringRqueueRedisTemplate.delete(queueDetail.getProcessingQueueName()); + stringRqueueRedisTemplate.delete(queueDetail.getDelayedQueueName()); + return true; + } catch (Exception e) { + log.error("Delete all message failed", e); + return false; + } + } + + @Override + public List getAllMessages(String queueName) { + List messages = new ArrayList<>(); + QueueDetail queueDetail = EndpointRegistry.get(queueName); + for (RqueueMessage message : + messageTemplate.getAllMessages( + queueDetail.getQueueName(), + queueDetail.getProcessingQueueName(), + queueDetail.getDelayedQueueName())) { + messages.add(MessageUtils.convertMessageToObject(message, messageConverter)); + } + return messages; + } + + @Override + public Object getMessage(String queueName, String id) { + RqueueMessage rqueueMessage = getRqueueMessage(queueName, id); + if (rqueueMessage == null) { + return null; + } + Message message = + MessageBuilder.createMessage( + rqueueMessage.getMessage(), RqueueMessageHeaders.emptyMessageHeaders()); + return messageConverter.fromMessage(message, null); + } + + @Override + public RqueueMessage getRqueueMessage(String queueName, String id) { + MessageMetadata messageMetadata = rqueueMessageMetadataService.getByMessageId(queueName, id); + if (messageMetadata == null) { + return null; + } + return messageMetadata.getRqueueMessage(); + } + + @Override + public boolean exist(String queueName, String id) { + if (rqueueLockManager.acquireLock(queueName, Duration.ofSeconds(1))) { + boolean exist = getMessage(queueName, id) != null; + rqueueLockManager.releaseLock(queueName); + return exist; + } + throw new LockCanNotBeAcquired(queueName); + } + + @Override + public boolean deleteMessage(String queueName, String id) { + RqueueMessage rqueueMessage = getRqueueMessage(queueName, id); + if (rqueueMessage == null) { + return false; + } + rqueueMessageMetadataService.deleteMessage( + queueName, id, Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute())); + return true; + } + + @Override + public MessageConverter getMessageConverter() { + return messageConverter; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java similarity index 51% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java index 07966923..b881d203 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java @@ -14,22 +14,20 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.core.impl; -import static com.github.sonus21.rqueue.utils.Constants.MIN_DELAY; -import static com.github.sonus21.rqueue.utils.MessageUtils.buildMessage; import static com.github.sonus21.rqueue.utils.Validator.validateDelay; import static com.github.sonus21.rqueue.utils.Validator.validateMessage; import static com.github.sonus21.rqueue.utils.Validator.validatePriority; import static com.github.sonus21.rqueue.utils.Validator.validateQueue; import static com.github.sonus21.rqueue.utils.Validator.validateRetryCount; import static org.springframework.util.Assert.isTrue; -import static org.springframework.util.Assert.notEmpty; import static org.springframework.util.Assert.notNull; -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; -import com.github.sonus21.rqueue.config.RqueueConfig; -import com.github.sonus21.rqueue.converter.GenericMessageConverter; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageSender; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.MessageMoveResult; import com.github.sonus21.rqueue.utils.Constants; @@ -39,43 +37,25 @@ import java.util.Collections; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.MessageConverter; -import org.springframework.messaging.converter.StringMessageConverter; @Slf4j -public class RqueueMessageSenderImpl implements RqueueMessageSender { - private final CompositeMessageConverter messageConverter; - private RqueueMessageTemplate messageTemplate; - @Autowired private RqueueRedisTemplate stringRqueueRedisTemplate; - @Autowired private RqueueConfig rqueueConfig; - - private RqueueMessageSenderImpl( - RqueueMessageTemplate messageTemplate, - List messageConverters, - boolean addDefault) { - notNull(messageTemplate, "messageTemplate cannot be null"); - notEmpty(messageConverters, "messageConverters cannot be empty"); - this.messageTemplate = messageTemplate; - this.messageConverter = - new CompositeMessageConverter(getMessageConverters(addDefault, messageConverters)); - } +public class RqueueMessageSenderImpl extends BaseMessageSender implements RqueueMessageSender { public RqueueMessageSenderImpl(RqueueMessageTemplate messageTemplate) { - this(messageTemplate, Collections.singletonList(new GenericMessageConverter()), false); + this(messageTemplate, Collections.emptyList()); } public RqueueMessageSenderImpl( RqueueMessageTemplate messageTemplate, List messageConverters) { - this(messageTemplate, messageConverters, false); + super(messageTemplate, messageConverters); } @Override public boolean enqueue(String queueName, Object message) { validateQueue(queueName); validateMessage(message); - return pushMessage(queueName, message, null, null); + return pushMessage(queueName, null, message, null, null) != null; } @Override @@ -83,7 +63,7 @@ public boolean enqueueWithRetry(String queueName, Object message, int retryCount validateQueue(queueName); validateMessage(message); validateRetryCount(retryCount); - return pushMessage(queueName, message, retryCount, null); + return pushMessage(queueName, null, message, retryCount, null) != null; } @Override @@ -92,7 +72,8 @@ public boolean enqueueWithPriority(String queueName, String priority, Object mes validatePriority(priority); validateMessage(message); return pushMessage( - PriorityUtils.getQueueNameForPriority(queueName, priority), message, null, null); + PriorityUtils.getQueueNameForPriority(queueName, priority), null, message, null, null) + != null; } @Override @@ -100,7 +81,7 @@ public boolean enqueueIn(String queueName, Object message, long delayInMilliSecs validateQueue(queueName); validateMessage(message); validateDelay(delayInMilliSecs); - return pushMessage(queueName, message, null, delayInMilliSecs); + return pushMessage(queueName, null, message, null, delayInMilliSecs) != null; } @Override @@ -110,7 +91,7 @@ public boolean enqueueInWithRetry( validateMessage(message); validateRetryCount(retryCount); validateDelay(delayInMilliSecs); - return pushMessage(queueName, message, retryCount, delayInMilliSecs); + return pushMessage(queueName, null, message, retryCount, delayInMilliSecs) != null; } @Override @@ -121,16 +102,18 @@ public boolean enqueueInWithPriority( validateMessage(message); validateDelay(delayInMilliSecs); return pushMessage( - PriorityUtils.getQueueNameForPriority(queueName, priority), - message, - null, - delayInMilliSecs); + PriorityUtils.getQueueNameForPriority(queueName, priority), + null, + message, + null, + delayInMilliSecs) + != null; } @Override public List getAllMessages(String queueName) { List messages = new ArrayList<>(); - QueueDetail queueDetail = QueueRegistry.get(queueName); + QueueDetail queueDetail = EndpointRegistry.get(queueName); for (RqueueMessage message : messageTemplate.getAllMessages( queueDetail.getQueueName(), @@ -179,77 +162,15 @@ private MessageMoveResult moveMessageListToList( @Override public boolean deleteAllMessages(String queueName) { - QueueDetail queueDetail = QueueRegistry.get(queueName); + QueueDetail queueDetail = EndpointRegistry.get(queueName); stringRqueueRedisTemplate.delete(queueDetail.getQueueName()); stringRqueueRedisTemplate.delete(queueDetail.getProcessingQueueName()); stringRqueueRedisTemplate.delete(queueDetail.getDelayedQueueName()); return true; } - private boolean pushMessage( - String queueName, Object message, Integer retryCount, Long delayInMilliSecs) { - QueueDetail queueDetail = QueueRegistry.get(queueName); - RqueueMessage rqueueMessage = - buildMessage( - messageConverter, queueDetail.getQueueName(), message, retryCount, delayInMilliSecs); - try { - if (delayInMilliSecs == null || delayInMilliSecs <= MIN_DELAY) { - messageTemplate.addMessage(queueDetail.getQueueName(), rqueueMessage); - } else { - messageTemplate.addMessageWithDelay( - queueDetail.getDelayedQueueName(), - queueDetail.getDelayedQueueChannelName(), - rqueueMessage); - } - } catch (Exception e) { - log.error("Queue: {} Message {} could not be pushed {}", queueName, rqueueMessage, e); - return false; - } - return true; - } - - private List getMessageConverters( - boolean addDefault, List messageConverters) { - List messageConverterList = new ArrayList<>(); - StringMessageConverter stringMessageConverter = new StringMessageConverter(); - stringMessageConverter.setSerializedPayloadClass(String.class); - messageConverterList.add(stringMessageConverter); - if (addDefault) { - messageConverterList.add(new GenericMessageConverter()); - } - messageConverterList.addAll(messageConverters); - return messageConverterList; - } - @Override public void registerQueue(String queueName, String... priorities) { - validateQueue(queueName); - notNull(priorities, "priorities cannot be null"); - QueueDetail queueDetail = - QueueDetail.builder() - .name(queueName) - .active(false) - .queueName(rqueueConfig.getQueueName(queueName)) - .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName)) - .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName)) - .processingQueueName(rqueueConfig.getProcessingQueueName(queueName)) - .processingQueueChannelName(rqueueConfig.getProcessingQueueChannelName(queueName)) - .build(); - QueueRegistry.register(queueDetail); - for (String priority : priorities) { - String suffix = PriorityUtils.getSuffix(priority); - queueDetail = - QueueDetail.builder() - .name(queueName + suffix) - .active(false) - .queueName(rqueueConfig.getQueueName(queueName) + suffix) - .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName) + suffix) - .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName) + suffix) - .processingQueueName(rqueueConfig.getProcessingQueueName(queueName) + suffix) - .processingQueueChannelName( - rqueueConfig.getProcessingQueueChannelName(queueName) + suffix) - .build(); - QueueRegistry.register(queueDetail); - } + registerQueueInternal(queueName, priorities); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java similarity index 94% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 7fdb0268..6045bbbd 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -14,12 +14,14 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.core.impl; import static com.github.sonus21.rqueue.core.RedisScriptFactory.getScript; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.core.RedisScriptFactory.ScriptType; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.models.MessageMoveResult; import com.github.sonus21.rqueue.utils.Constants; import java.util.ArrayList; @@ -58,7 +60,7 @@ public RqueueMessage pop( long visibilityTimeout) { long currentTime = System.currentTimeMillis(); RedisScript script = - (RedisScript) getScript(ScriptType.POP_MESSAGE); + (RedisScript) getScript(ScriptType.DEQUEUE_MESSAGE); return scriptExecutor.execute( script, Arrays.asList(queueName, processingQueueName, processingChannelName), @@ -70,7 +72,7 @@ public RqueueMessage pop( public Long addMessageWithDelay( String delayQueueName, String delayQueueChannelName, RqueueMessage rqueueMessage) { long queuedTime = rqueueMessage.getQueuedTime(); - RedisScript script = (RedisScript) getScript(ScriptType.ADD_MESSAGE); + RedisScript script = (RedisScript) getScript(ScriptType.ENQUEUE_MESSAGE); return scriptExecutor.execute( script, Arrays.asList(delayQueueName, delayQueueChannelName), @@ -210,6 +212,15 @@ public List> readFromZsetWithScore(String name, long s return new ArrayList<>(messages); } + @Override + public Long getScore(String delayedQueueName, RqueueMessage rqueueMessage) { + Double score = redisTemplate.opsForZSet().score(delayedQueueName, rqueueMessage); + if (score == null) { + return null; + } + return score.longValue(); + } + @Override public List readFromList(String name, long start, long end) { List messages = lrange(name, start, end); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageFactory.java index abd17ea9..aa116f8c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageFactory.java @@ -16,19 +16,17 @@ package com.github.sonus21.rqueue.core.support; -import com.github.sonus21.rqueue.converter.GenericMessageConverter; import com.github.sonus21.rqueue.core.RqueueMessage; import java.util.ArrayList; import java.util.List; import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; -public class RqueueMessageFactory { - private static final GenericMessageConverter converter = new GenericMessageConverter(); - - RqueueMessageFactory() {} +public final class RqueueMessageFactory { + private RqueueMessageFactory() {} public static RqueueMessage buildMessage( - Object object, String queueName, Integer retryCount, Long delay) { + MessageConverter converter, Object object, String queueName, Integer retryCount, Long delay) { Object payload = object; if (object == null) { payload = "Test message"; @@ -38,19 +36,26 @@ public static RqueueMessage buildMessage( return new RqueueMessage(queueName, (String) msg.getPayload(), retryCount, delay); } - public static List generateMessages(String queueName, int count) { - return generateMessages(null, queueName, null, null, count); + public static List generateMessages( + MessageConverter converter, String queueName, int count) { + return generateMessages(converter, null, queueName, null, null, count); } - public static List generateMessages(String queueName, long delay, int count) { - return generateMessages(null, queueName, null, delay, count); + public static List generateMessages( + MessageConverter converter, String queueName, long delay, int count) { + return generateMessages(converter, null, queueName, null, delay, count); } public static List generateMessages( - Object object, String queueName, Integer retryCount, Long delay, int count) { + MessageConverter converter, + Object object, + String queueName, + Integer retryCount, + Long delay, + int count) { List messages = new ArrayList<>(); for (int i = 0; i < count; i++) { - messages.add(buildMessage(object, queueName, retryCount, delay)); + messages.add(buildMessage(converter, object, queueName, retryCount, delay)); } return messages; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/LockCanNotBeAcquired.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/LockCanNotBeAcquired.java new file mode 100644 index 00000000..5336be39 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/LockCanNotBeAcquired.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.exception; + +/** + * Whenever a Redis lock can not be acuired due to some other is holding lock then this error would + * be thrown. The application should retry once this error occurs. + */ +public class LockCanNotBeAcquired extends RuntimeException { + + private static final long serialVersionUID = 598739372785907190L; + + public LockCanNotBeAcquired(String name) { + super(name); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/QueueDoesNotExist.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/QueueDoesNotExist.java index 7926263c..3f9000b6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/QueueDoesNotExist.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/QueueDoesNotExist.java @@ -16,6 +16,11 @@ package com.github.sonus21.rqueue.exception; +/** + * This exception is raised, when a queue is not registered in {@link + * com.github.sonus21.rqueue.core.EndpointRegistry}, to register a queue use {@link + * com.github.sonus21.rqueue.core.RqueueEndpointManager#registerQueue(String, String...)} + */ public class QueueDoesNotExist extends RuntimeException { private static final long serialVersionUID = 598739372785907190L; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java index e1c4a765..2fddbccc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.listener; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.utils.ThreadUtils.QueueThread; import com.github.sonus21.rqueue.utils.TimeoutUtils; import java.util.Collections; @@ -30,8 +31,8 @@ class DefaultRqueuePoller extends RqueueMessagePoller { QueueDetail queueDetail, RqueueMessageListenerContainer container, PostProcessingHandler postProcessingHandler, - int retryPerPoll) { - super(queueDetail.getName(), container, postProcessingHandler, retryPerPoll); + RqueueConfig rqueueConfig) { + super(queueDetail.getName(), container, postProcessingHandler, rqueueConfig); this.queueDetail = queueDetail; this.queueThread = queueThread; this.queues = Collections.singletonList(queueDetail.getName()); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java index a714fdc1..7c9cba79 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java @@ -18,19 +18,19 @@ import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.support.MessageProcessor; -import com.github.sonus21.rqueue.models.db.TaskStatus; +import com.github.sonus21.rqueue.models.db.MessageStatus; import com.github.sonus21.rqueue.utils.BaseLogger; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; @Slf4j -public class MessageProcessorHandler extends BaseLogger { +class MessageProcessorHandler extends BaseLogger { private final MessageProcessor manualDeletionMessageProcessor; private final MessageProcessor deadLetterQueueMessageProcessor; private final MessageProcessor discardMessageProcessor; private final MessageProcessor postExecutionMessageProcessor; - public MessageProcessorHandler( + MessageProcessorHandler( MessageProcessor manualDeletionMessageProcessor, MessageProcessor deadLetterQueueMessageProcessor, MessageProcessor discardMessageProcessor, @@ -42,7 +42,7 @@ public MessageProcessorHandler( this.postExecutionMessageProcessor = postExecutionMessageProcessor; } - public void handleMessage(RqueueMessage rqueueMessage, Object userMessage, TaskStatus status) { + void handleMessage(RqueueMessage rqueueMessage, Object userMessage, MessageStatus status) { MessageProcessor messageProcessor = null; switch (status) { case DELETED: diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java index 5d819846..5effad7b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java @@ -16,19 +16,17 @@ package com.github.sonus21.rqueue.listener; -import static com.github.sonus21.rqueue.utils.Constants.SECONDS_IN_A_WEEK; - import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.exception.UnknownSwitchCase; import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.models.db.MessageStatus; import com.github.sonus21.rqueue.models.db.QueueConfig; -import com.github.sonus21.rqueue.models.db.TaskStatus; +import com.github.sonus21.rqueue.models.enums.ExecutionStatus; import com.github.sonus21.rqueue.models.event.RqueueExecutionEvent; import com.github.sonus21.rqueue.utils.BaseLogger; -import com.github.sonus21.rqueue.utils.MessageUtils; import com.github.sonus21.rqueue.utils.RedisUtils; import com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff; import com.github.sonus21.rqueue.web.dao.RqueueSystemConfigDao; @@ -70,21 +68,20 @@ class PostProcessingHandler extends BaseLogger { this.rqueueConfig = rqueueConfig; } - void handlePostProcessing( + void handle( QueueDetail queueDetail, RqueueMessage rqueueMessage, Object userMessage, MessageMetadata messageMetadata, - TaskStatus status, + ExecutionStatus status, int failureCount, long jobExecutionStartTime) { - if (status == TaskStatus.QUEUE_INACTIVE) { - return; - } try { switch (status) { - case SUCCESSFUL: - handleSuccessFullExecution( + case QUEUE_INACTIVE: + return; + case DELETED: + handleManualDeletion( queueDetail, rqueueMessage, userMessage, @@ -92,8 +89,8 @@ void handlePostProcessing( failureCount, jobExecutionStartTime); break; - case DELETED: - handleManualDeletion( + case IGNORED: + handleIgnoredMessage( queueDetail, rqueueMessage, userMessage, @@ -101,8 +98,11 @@ void handlePostProcessing( failureCount, jobExecutionStartTime); break; - case IGNORED: - handleIgnoredMessage( + case OLD_MESSAGE: + handleOldMessage(queueDetail, rqueueMessage); + break; + case SUCCESSFUL: + handleSuccessFullExecution( queueDetail, rqueueMessage, userMessage, @@ -123,50 +123,59 @@ void handlePostProcessing( throw new UnknownSwitchCase(String.valueOf(status)); } } catch (Exception e) { - log(Level.ERROR, "Error occurred in post processing", e); + log( + Level.ERROR, + "Error occurred in post processing, RqueueMessage: {}, Status: {}", + e, + rqueueMessage, + status); } } + private void handleOldMessage(QueueDetail queueDetail, RqueueMessage rqueueMessage) { + if (isDebugEnabled()) { + log( + Level.DEBUG, + "Message {} ignored due to new message, Queue: {}", + null, + rqueueMessage, + queueDetail.getName()); + } + rqueueMessageTemplate.removeElementFromZset( + queueDetail.getProcessingQueueName(), rqueueMessage); + } + private void publishEvent( QueueDetail queueDetail, RqueueMessage rqueueMessage, MessageMetadata messageMetadata, - TaskStatus status, + MessageStatus status, long jobExecutionStartTime) { + updateMetadata(messageMetadata, rqueueMessage, jobExecutionStartTime, status); if (rqueueWebConfig.isCollectListenerStats()) { - MessageMetadata newMessageMetaData = - addOrDeleteMetadata(rqueueMessage, messageMetadata, jobExecutionStartTime, false); RqueueExecutionEvent event = - new RqueueExecutionEvent(queueDetail, rqueueMessage, status, newMessageMetaData); + new RqueueExecutionEvent( + queueDetail, rqueueMessage, status.getTaskStatus(), messageMetadata); applicationEventPublisher.publishEvent(event); } } - private MessageMetadata addOrDeleteMetadata( - RqueueMessage rqueueMessage, + private void updateMetadata( MessageMetadata messageMetadata, + RqueueMessage rqueueMessage, long jobExecutionStartTime, - boolean saveOrDelete) { - MessageMetadata newMessageMetaData = messageMetadata; - String messageMetadataId = MessageUtils.getMessageMetaId(rqueueMessage.getId()); - if (newMessageMetaData == null) { - newMessageMetaData = rqueueMessageMetadataService.get(messageMetadataId); - } - if (newMessageMetaData == null) { - newMessageMetaData = new MessageMetadata(messageMetadataId, rqueueMessage.getId()); - // do not call db delete method - if (!saveOrDelete) { - newMessageMetaData.addExecutionTime(jobExecutionStartTime); - return newMessageMetaData; - } - } - newMessageMetaData.addExecutionTime(jobExecutionStartTime); - if (saveOrDelete) { - rqueueMessageMetadataService.save(newMessageMetaData, Duration.ofSeconds(SECONDS_IN_A_WEEK)); + MessageStatus messageStatus) { + messageMetadata.setStatus(messageStatus); + messageMetadata.setRqueueMessage(rqueueMessage); + messageMetadata.addExecutionTime(jobExecutionStartTime); + if (messageStatus.isTerminalState()) { + rqueueMessageMetadataService.save( + messageMetadata, + Duration.ofSeconds(rqueueConfig.getMessageDurabilityInTerminalStateInSecond())); } else { - rqueueMessageMetadataService.delete(messageMetadataId); + rqueueMessageMetadataService.save( + messageMetadata, Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute())); } - return newMessageMetaData; } private void deleteMessage( @@ -174,7 +183,7 @@ private void deleteMessage( RqueueMessage rqueueMessage, Object userMessage, MessageMetadata messageMetadata, - TaskStatus status, + MessageStatus status, int failureCount, long jobExecutionStartTime) { rqueueMessageTemplate.removeElementFromZset( @@ -207,7 +216,7 @@ private void moveMessageForReprocessingOrDlq( RqueueMessage oldMessage, RqueueMessage newMessage, Object userMessage) { - messageProcessorHandler.handleMessage(newMessage, userMessage, TaskStatus.MOVED_TO_DLQ); + messageProcessorHandler.handleMessage(newMessage, userMessage, MessageStatus.MOVED_TO_DLQ); if (queueDetail.isDeadLetterConsumerEnabled()) { String configKey = rqueueConfig.getQueueConfigKey(queueDetail.getDeadLetterQueueName()); QueueConfig queueConfig = rqueueSystemConfigDao.getQConfig(configKey, true); @@ -249,9 +258,9 @@ private void moveMessageToDlq( moveMessageForReprocessingOrDlq(queueDetail, rqueueMessage, newMessage, userMessage); publishEvent( queueDetail, - rqueueMessage, + newMessage, messageMetadata, - TaskStatus.MOVED_TO_DLQ, + MessageStatus.MOVED_TO_DLQ, jobExecutionStartTime); } @@ -276,7 +285,7 @@ private void parkMessageForRetry( rqueueMessage, newMessage, delay); - addOrDeleteMetadata(rqueueMessage, messageMetadata, jobExecutionStartTime, true); + updateMetadata(messageMetadata, newMessage, jobExecutionStartTime, MessageStatus.FAILED); } private void discardMessage( @@ -294,7 +303,7 @@ private void discardMessage( rqueueMessage, userMessage, messageMetadata, - TaskStatus.DISCARDED, + MessageStatus.DISCARDED, failureCount, jobExecutionStartTime); } @@ -314,7 +323,7 @@ private void handleManualDeletion( rqueueMessage, userMessage, messageMetadata, - TaskStatus.DELETED, + MessageStatus.DELETED, failureCount, jobExecutionStartTime); } @@ -334,7 +343,7 @@ private void handleSuccessFullExecution( rqueueMessage, userMessage, messageMetadata, - TaskStatus.SUCCESSFUL, + MessageStatus.SUCCESSFUL, failureCount, jobExecutionStartTime); } @@ -427,7 +436,7 @@ private void handleIgnoredMessage( rqueueMessage, userMessage, messageMetadata, - TaskStatus.IGNORED, + MessageStatus.IGNORED, failureCount, jobExecutionStartTime); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java index 28b97642..3eb4ba10 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java @@ -77,7 +77,7 @@ public QueueConfig toConfig() { .updatedOn(System.currentTimeMillis()) .deadLetterQueues(new LinkedList<>()) .concurrency(concurrency.toMinMax()) - .priority(priority) + .priority(Collections.unmodifiableMap(priority)) .priorityGroup(priorityGroup) .systemGenerated(systemGenerated) .build(); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index 9771a361..e65a45e8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -18,13 +18,16 @@ import static com.github.sonus21.rqueue.utils.Constants.DELTA_BETWEEN_RE_ENQUEUE_TIME; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.metrics.RqueueMetricsCounter; import com.github.sonus21.rqueue.models.db.MessageMetadata; -import com.github.sonus21.rqueue.models.db.TaskStatus; +import com.github.sonus21.rqueue.models.db.MessageStatus; +import com.github.sonus21.rqueue.models.enums.ExecutionStatus; import com.github.sonus21.rqueue.utils.MessageUtils; import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.Objects; import java.util.concurrent.Semaphore; import lombok.extern.slf4j.Slf4j; @@ -36,32 +39,37 @@ @Slf4j class RqueueExecutor extends MessageContainerBase { private final QueueDetail queueDetail; - private final Message message; private final RqueueMessage rqueueMessage; private final RqueueMessageHandler rqueueMessageHandler; private final RqueueMessageMetadataService rqueueMessageMetadataService; private final PostProcessingHandler postProcessingHandler; - private final String messageMetadataId; private final Semaphore semaphore; - private final int retryPerPoll; + private final RqueueConfig rqueueConfig; + private Message message; private MessageMetadata messageMetadata; private Object userMessage; + private boolean updatedToProcessing; RqueueExecutor( RqueueMessage rqueueMessage, QueueDetail queueDetail, Semaphore semaphore, WeakReference container, - int retryPerPoll, + RqueueConfig rqueueConfig, PostProcessingHandler postProcessingHandler) { super(log, queueDetail.getName(), container); this.rqueueMessage = rqueueMessage; this.queueDetail = queueDetail; this.semaphore = semaphore; this.rqueueMessageHandler = Objects.requireNonNull(container.get()).getRqueueMessageHandler(); - this.messageMetadataId = MessageUtils.getMessageMetaId(rqueueMessage.getId()); - this.retryPerPoll = retryPerPoll; + this.rqueueConfig = rqueueConfig; this.postProcessingHandler = postProcessingHandler; + this.rqueueMessageMetadataService = + Objects.requireNonNull(container.get()).getRqueueMessageMetadataService(); + init(); + } + + private void init() { this.message = MessageBuilder.createMessage( rqueueMessage.getMessage(), @@ -71,9 +79,9 @@ class RqueueExecutor extends MessageContainerBase { MessageUtils.convertMessageToObject(message, rqueueMessageHandler.getMessageConverter()); } catch (Exception e) { log(Level.ERROR, "Unable to convert message {}", e, rqueueMessage.getMessage()); + throw e; } - this.rqueueMessageMetadataService = - Objects.requireNonNull(container.get()).getRqueueMessageMetadataService(); + this.messageMetadata = rqueueMessageMetadataService.getOrCreateMessageMetadata(rqueueMessage); } private int getMaxRetryCount() { @@ -102,11 +110,11 @@ private long getMaxProcessingTime() { } private boolean isMessageDeleted() { - messageMetadata = rqueueMessageMetadataService.get(messageMetadataId); - if (messageMetadata == null) { - return false; + if (this.messageMetadata.isDeleted()) { + return true; } - return messageMetadata.isDeleted(); + this.messageMetadata = rqueueMessageMetadataService.getOrCreateMessageMetadata(rqueueMessage); + return this.messageMetadata.isDeleted(); } private boolean shouldIgnore() { @@ -115,64 +123,86 @@ private boolean shouldIgnore() { .process(userMessage, rqueueMessage); } + private boolean isOldMessage() { + return messageMetadata.getRqueueMessage() != null + && messageMetadata.getRqueueMessage().getQueuedTime() != rqueueMessage.getQueuedTime(); + } + private int getRetryCount() { int maxRetry = getMaxRetryCount(); - if (retryPerPoll == -1) { + if (rqueueConfig.getRetryPerPoll() == -1) { return maxRetry; } - return Math.min(retryPerPoll, maxRetry); + return Math.min(rqueueConfig.getRetryPerPoll(), maxRetry); } - private boolean queueActive() { - return isQueueActive(queueDetail.getName()); + private boolean queueInActive() { + return !isQueueActive(queueDetail.getName()); } - private TaskStatus getStatus() { - if (!queueActive()) { - return TaskStatus.QUEUE_INACTIVE; + private ExecutionStatus getStatus() { + if (queueInActive()) { + return ExecutionStatus.QUEUE_INACTIVE; } if (shouldIgnore()) { - return TaskStatus.IGNORED; + return ExecutionStatus.IGNORED; + } + if (isOldMessage()) { + return ExecutionStatus.OLD_MESSAGE; } if (isMessageDeleted()) { - return TaskStatus.DELETED; + return ExecutionStatus.DELETED; } return null; } + private void updateToProcessing() { + if (updatedToProcessing) { + return; + } + this.updatedToProcessing = true; + messageMetadata.setStatus(MessageStatus.PROCESSING); + this.rqueueMessageMetadataService.save( + messageMetadata, Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute())); + } + @Override void start() { int failureCount = rqueueMessage.getFailureCount(); long maxProcessingTime = getMaxProcessingTime(); long startTime = System.currentTimeMillis(); int retryCount = getRetryCount(); - TaskStatus status; + int attempt = 1; + ExecutionStatus status; try { do { + log(Level.DEBUG, "Attempt {} message: {}", null, attempt, userMessage); status = getStatus(); if (status != null) { break; } try { + updateToProcessing(); updateCounter(false); rqueueMessageHandler.handleMessage(message); - status = TaskStatus.SUCCESSFUL; + status = ExecutionStatus.SUCCESSFUL; } catch (MessagingException e) { updateCounter(true); failureCount += 1; } catch (Exception e) { updateCounter(true); failureCount += 1; - log(Level.ERROR, "Message execution failed", e); + log(Level.ERROR, "Message execution failed, RqueueMessage: {}", e, rqueueMessage); } - retryCount--; + retryCount -= 1; + attempt += 1; } while (retryCount > 0 && status == null && System.currentTimeMillis() < maxProcessingTime); - postProcessingHandler.handlePostProcessing( + postProcessingHandler.handle( queueDetail, rqueueMessage, userMessage, messageMetadata, - status == null ? TaskStatus.FAILED : status, + (status == null ? ExecutionStatus.FAILED : status), failureCount, startTime); } finally { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java index e9abed57..755beab6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java @@ -19,8 +19,8 @@ import static org.springframework.util.Assert.notNull; import com.github.sonus21.rqueue.annotation.RqueueListener; -import com.github.sonus21.rqueue.converter.GenericMessageConverter; -import com.github.sonus21.rqueue.core.QueueRegistry; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.MessageConverterFactory; import com.github.sonus21.rqueue.exception.QueueDoesNotExist; import com.github.sonus21.rqueue.models.Concurrency; import com.github.sonus21.rqueue.utils.Constants; @@ -44,7 +44,6 @@ import org.springframework.format.support.DefaultFormattingConversionService; import org.springframework.messaging.Message; import org.springframework.messaging.MessagingException; -import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.handler.HandlerMethod; import org.springframework.messaging.handler.annotation.support.AnnotationExceptionHandlerMethodResolver; @@ -60,7 +59,7 @@ import org.springframework.util.comparator.ComparableComparator; public class RqueueMessageHandler extends AbstractMethodMessageHandler { - private ConversionService conversionService = new DefaultFormattingConversionService(); + private final ConversionService conversionService; private final MessageConverter messageConverter; public RqueueMessageHandler() { @@ -69,9 +68,8 @@ public RqueueMessageHandler() { public RqueueMessageHandler(List messageConverters) { notNull(messageConverters, "messageConverters cannot be null"); - List messageConverterList = new ArrayList<>(messageConverters); - messageConverterList.add(new GenericMessageConverter()); - this.messageConverter = new CompositeMessageConverter(messageConverterList); + this.messageConverter = MessageConverterFactory.getMessageConverter(messageConverters); + this.conversionService = new DefaultFormattingConversionService(); } private ConfigurableBeanFactory getBeanFactory() { @@ -286,9 +284,9 @@ protected MappingInformation getMatchingMapping(MappingInformation mapping, Mess return mapping; } try { - QueueDetail queueDetail = QueueRegistry.get(destination); + QueueDetail queueDetail = EndpointRegistry.get(destination); if (queueDetail.isSystemGenerated()) { - queueDetail = QueueRegistry.get(queueDetail.getPriorityGroup()); + queueDetail = EndpointRegistry.get(queueDetail.getPriorityGroup()); if (mapping.getQueueNames().contains(queueDetail.getName())) { return mapping; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java index 7689b772..7d9b7c91 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java @@ -17,16 +17,22 @@ package com.github.sonus21.rqueue.listener; import com.github.sonus21.rqueue.core.RqueueMessage; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.springframework.messaging.MessageHeaders; public class RqueueMessageHeaders { - private RqueueMessageHeaders() {} - public static final String DESTINATION = "destination"; public static final String ID = "messageId"; public static final String MESSAGE = "message"; + private static final MessageHeaders emptyMessageHeaders = + new MessageHeaders(Collections.emptyMap()); + private RqueueMessageHeaders() {} + + public static MessageHeaders emptyMessageHeaders() { + return emptyMessageHeaders; + } static MessageHeaders buildMessageHeaders(String destination, RqueueMessage rqueueMessage) { Map headers = new HashMap<>(3); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java index 810d3fc9..f8e0769a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java @@ -23,12 +23,13 @@ import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueWebConfig; -import com.github.sonus21.rqueue.core.QueueRegistry; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.metrics.RqueueMetricsCounter; import com.github.sonus21.rqueue.models.Concurrency; import com.github.sonus21.rqueue.models.enums.PriorityMode; +import com.github.sonus21.rqueue.models.enums.RqueueMode; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.StringUtils; @@ -240,16 +241,20 @@ public void stop(Runnable callback) { @Override public void afterPropertiesSet() throws Exception { synchronized (lifecycleMgr) { - QueueRegistry.delete(); + if (RqueueMode.PRODUCER.equals(rqueueConfig.getMode())) { + log.info("Producer only mode running."); + return; + } + EndpointRegistry.delete(); for (MappingInformation mappingInformation : rqueueMessageHandler.getHandlerMethods().keySet()) { for (String queue : mappingInformation.getQueueNames()) { for (QueueDetail queueDetail : getQueueDetail(queue, mappingInformation)) { - QueueRegistry.register(queueDetail); + EndpointRegistry.register(queueDetail); } } } - List queueDetails = QueueRegistry.getActiveQueueDetails(); + List queueDetails = EndpointRegistry.getActiveQueueDetails(); if (queueDetails.isEmpty()) { return; } @@ -278,7 +283,7 @@ private void initializeThreadMap( } private void initializeRunningQueueState() { - for (String queue : QueueRegistry.getActiveQueues()) { + for (String queue : EndpointRegistry.getActiveQueues()) { queueRunningState.put(queue, false); } } @@ -409,7 +414,7 @@ private void createPostProcessingHandler() { protected void doStart() { Map> queueGroupToDetails = new HashMap<>(); - for (QueueDetail queueDetail : QueueRegistry.getActiveQueueDetails()) { + for (QueueDetail queueDetail : EndpointRegistry.getActiveQueueDetails()) { int prioritySize = queueDetail.getPriority().size(); if (prioritySize == 0) { startQueue(queueDetail.getName(), queueDetail); @@ -455,7 +460,7 @@ protected void startGroup(String groupName, List queueDetails) { queueDetails, queueThread, postProcessingHandler, - rqueueConfig.getRetryPerPoll())); + rqueueConfig)); } else { future = taskExecutor.submit( @@ -465,7 +470,7 @@ protected void startGroup(String groupName, List queueDetails) { queueDetails, queueThread, postProcessingHandler, - rqueueConfig.getRetryPerPoll())); + rqueueConfig)); } scheduledFutureByQueue.put(groupName, future); } @@ -478,7 +483,7 @@ protected void startQueue(String queueName, QueueDetail queueDetail) { QueueThread queueThread = queueThreadMap.get(queueName); DefaultRqueuePoller messagePoller = new DefaultRqueuePoller( - queueThread, queueDetail, this, postProcessingHandler, rqueueConfig.getRetryPerPoll()); + queueThread, queueDetail, this, postProcessingHandler, rqueueConfig); Future future = getTaskExecutor().submit(messagePoller); scheduledFutureByQueue.put(queueName, future); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java index fa155c06..2caeb8c3 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.listener; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.utils.ThreadUtils.QueueThread; import java.util.List; @@ -28,17 +29,17 @@ @Slf4j abstract class RqueueMessagePoller extends MessageContainerBase { private final PostProcessingHandler postProcessingHandler; - private final int retryPerPoll; + private final RqueueConfig rqueueConfig; List queues; RqueueMessagePoller( String groupName, RqueueMessageListenerContainer container, PostProcessingHandler postProcessingHandler, - int retryPerPoll) { + RqueueConfig rqueueConfig) { super(log, groupName, container); this.postProcessingHandler = postProcessingHandler; - this.retryPerPoll = retryPerPoll; + this.rqueueConfig = rqueueConfig; } private RqueueMessage getMessage(QueueDetail queueDetail) { @@ -58,7 +59,7 @@ long getBackOffTime() { return Objects.requireNonNull(container.get()).getBackOffTime(); } - private void enqueue(QueueThread queueThread, QueueDetail queueDetail, RqueueMessage message) { + private void execute(QueueThread queueThread, QueueDetail queueDetail, RqueueMessage message) { queueThread .getTaskExecutor() .execute( @@ -67,7 +68,7 @@ private void enqueue(QueueThread queueThread, QueueDetail queueDetail, RqueueMes queueDetail, queueThread.getSemaphore(), container, - retryPerPoll, + rqueueConfig, postProcessingHandler)); } @@ -99,7 +100,7 @@ void poll(int index, String queue, QueueDetail queueDetail, QueueThread queueThr RqueueMessage message = getMessage(queueDetail); log(Level.DEBUG, "Queue: {} Fetched Msg {}", null, queue, message); if (message != null) { - enqueue(queueThread, queueDetail, message); + execute(queueThread, queueDetail, message); } else { semaphore.release(); deactivate(index, queue, DeactivateType.NO_MESSAGE); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java index 57468b47..9df787f8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java @@ -18,6 +18,7 @@ import static com.github.sonus21.rqueue.utils.Constants.BLANK; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.ThreadUtils.QueueThread; import com.github.sonus21.rqueue.utils.TimeoutUtils; @@ -41,8 +42,8 @@ class StrictPriorityPoller extends RqueueMessagePoller { final List queueDetails, final Map queueNameToThread, PostProcessingHandler postProcessingHandler, - int retryPerPoll) { - super("Strict-" + groupName, container, postProcessingHandler, retryPerPoll); + RqueueConfig rqueueConfig) { + super("Strict-" + groupName, container, postProcessingHandler, rqueueConfig); List queueDetailList = new ArrayList<>(queueDetails); queueDetailList.sort( (o1, o2) -> diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java index 75e95e44..5b649372 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.listener; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.ThreadUtils.QueueThread; import com.github.sonus21.rqueue.utils.TimeoutUtils; @@ -41,8 +42,8 @@ class WeightedPriorityPoller extends RqueueMessagePoller { final List queueDetails, final Map queueNameToThread, PostProcessingHandler postProcessingHandler, - int retryPerPoll) { - super("Weighted-" + groupName, container, postProcessingHandler, retryPerPoll); + RqueueConfig rqueueConfig) { + super("Weighted-" + groupName, container, postProcessingHandler, rqueueConfig); this.queueDetailList = queueDetails; this.queues = queueDetails.stream().map(QueueDetail::getName).collect(Collectors.toList()); this.queueNameToDetail = diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java index d27da8ca..90647f62 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.MetricsProperties; -import com.github.sonus21.rqueue.core.QueueRegistry; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import io.micrometer.core.instrument.Gauge; @@ -65,7 +65,7 @@ private long size(String name, boolean isZset) { } private void monitor() { - for (QueueDetail queueDetail : QueueRegistry.getActiveQueueDetails()) { + for (QueueDetail queueDetail : EndpointRegistry.getActiveQueueDetails()) { Tags queueTags = Tags.concat(metricsProperties.getMetricTags(), "queue", queueDetail.getName()); Gauge.builder(QUEUE_SIZE, queueDetail, c -> size(queueDetail.getQueueName(), false)) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java index 6d440317..2926bd98 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.models.db; +import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.models.SerializableBase; import com.github.sonus21.rqueue.utils.MessageUtils; import lombok.AllArgsConstructor; @@ -32,19 +33,21 @@ public class MessageMetadata extends SerializableBase { private static final long serialVersionUID = 4200184682879443328L; private String id; - private String messageId; private long totalExecutionTime; private boolean deleted; private Long deletedOn; + private RqueueMessage rqueueMessage; + private MessageStatus status; - public MessageMetadata(String id, String messageId) { + public MessageMetadata(String id, MessageStatus messageStatus) { this.id = id; - this.messageId = messageId; + this.status = messageStatus; } - public MessageMetadata(String messageId) { - this.id = MessageUtils.getMessageMetaId(messageId); - this.messageId = messageId; + public MessageMetadata(RqueueMessage rqueueMessage, MessageStatus messageStatus) { + this.id = MessageUtils.getMessageMetaId(rqueueMessage.getQueueName(), rqueueMessage.getId()); + this.rqueueMessage = rqueueMessage; + this.status = messageStatus; } public void addExecutionTime(long jobStartTime) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageStatus.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageStatus.java new file mode 100644 index 00000000..52c138b3 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageStatus.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.models.db; + +import com.github.sonus21.rqueue.models.enums.TaskStatus; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Getter +public enum MessageStatus { + // Message is just enqueued + ENQUEUED(false, null), + // Currently this message is being processed + PROCESSING(false, null), + // Message was deleted + DELETED(true, TaskStatus.DELETED), + // Message was ignored by pre processor + IGNORED(true, TaskStatus.IGNORED), + // Message was successful consumed + SUCCESSFUL(true, TaskStatus.SUCCESSFUL), + // Message moved to dead letter queue + MOVED_TO_DLQ(true, TaskStatus.MOVED_TO_DLQ), + /** + * Message was discarded due to retry limit or {@link + * com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff#STOP} was returned by task + * execution backoff method. + */ + DISCARDED(true, TaskStatus.DISCARDED), + // Execution has failed, it will retried later + FAILED(false, null); + private final boolean terminalState; + private final TaskStatus taskStatus; +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java index 8e159b0f..a5242210 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java @@ -130,20 +130,24 @@ public boolean updatePriority(Map newPriority) { return false; } boolean updated = false; + Map updatedPriority = new HashMap<>(priority); for (Entry entry : newPriority.entrySet()) { Integer val = priority.get(entry.getKey()); if (val == null || !val.equals(entry.getValue())) { updated = true; - priority.put(entry.getKey(), entry.getValue()); + updatedPriority.put(entry.getKey(), entry.getValue()); } } for (String key : priority.keySet()) { Integer val = newPriority.get(key); if (val == null) { updated = true; - priority.remove(key); + updatedPriority.remove(key); } } + if (updated) { + this.priority = updatedPriority; + } return updated; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/ExecutionStatus.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/ExecutionStatus.java new file mode 100644 index 00000000..63df9368 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/ExecutionStatus.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.models.enums; + +public enum ExecutionStatus { + SUCCESSFUL, + DELETED, + FAILED, + IGNORED, + OLD_MESSAGE, + QUEUE_INACTIVE, +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/RqueueMode.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/RqueueMode.java new file mode 100644 index 00000000..0d65a7a9 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/RqueueMode.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.models.enums; + +public enum RqueueMode { + PRODUCER, + CONSUMER, + BOTH, +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/TaskStatus.java similarity index 84% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/TaskStatus.java index 486174b9..7ace0464 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/TaskStatus.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.models.db; +package com.github.sonus21.rqueue.models.enums; import java.util.Arrays; import java.util.List; @@ -26,16 +26,15 @@ @AllArgsConstructor(access = AccessLevel.PRIVATE) @Getter public enum TaskStatus { - MOVED_TO_DLQ("Moved to dead letter queue messages", true), - SUCCESSFUL("Successful execution", true), + IGNORED("Ignored task", false), DELETED("Message deleted", false), + SUCCESSFUL("Successful execution", true), DISCARDED("Message discarded", true), - RETRIED("Retired at least once", true), - FAILED("failed", false), - IGNORED("Ignored task", false), - QUEUE_INACTIVE("Queue inactive", false); - private String description; - private boolean chartEnabled; + MOVED_TO_DLQ("Moved to dead letter queue messages", true), + RETRIED("Retired at least once", true); + + private final String description; + private final boolean chartEnabled; public static List getActiveChartStatus() { return Arrays.stream(values()).filter(TaskStatus::isChartEnabled).collect(Collectors.toList()); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueueExecutionEvent.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueueExecutionEvent.java index f89479dd..46658751 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueueExecutionEvent.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueueExecutionEvent.java @@ -19,7 +19,7 @@ import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.MessageMetadata; -import com.github.sonus21.rqueue.models.db.TaskStatus; +import com.github.sonus21.rqueue.models.enums.TaskStatus; import lombok.Getter; import org.springframework.context.ApplicationEvent; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java index 1f9fbc12..9955f939 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java @@ -16,6 +16,9 @@ package com.github.sonus21.rqueue.utils; +import com.github.sonus21.rqueue.config.RqueueConfig; +import java.net.InetSocketAddress; +import java.net.Proxy; import lombok.extern.slf4j.Slf4j; import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; @@ -25,13 +28,24 @@ public class HttpUtils { private HttpUtils() {} - public static T readUrl(String url, Class clazz) { + private static SimpleClientHttpRequestFactory getRequestFactory(RqueueConfig rqueueConfig) { + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setReadTimeout(2 * Constants.ONE_MILLI_INT); + requestFactory.setConnectTimeout(2 * Constants.ONE_MILLI_INT); + if (StringUtils.isEmpty(rqueueConfig.getProxyHost())) { + return requestFactory; + } + Proxy proxy = + new Proxy( + rqueueConfig.getProxyType(), + new InetSocketAddress(rqueueConfig.getProxyHost(), rqueueConfig.getProxyPort())); + requestFactory.setProxy(proxy); + return requestFactory; + } + + public static T readUrl(RqueueConfig rqueueConfig, String url, Class clazz) { try { - RestTemplate restTemplate = new RestTemplate(); - SimpleClientHttpRequestFactory rf = - (SimpleClientHttpRequestFactory) restTemplate.getRequestFactory(); - rf.setReadTimeout(2 * Constants.ONE_MILLI_INT); - rf.setConnectTimeout(2 * Constants.ONE_MILLI_INT); + RestTemplate restTemplate = new RestTemplate(getRequestFactory(rqueueConfig)); return restTemplate.getForObject(url, clazz); } catch (Exception e) { log.error("GET call failed for {}", url, e); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java index 7c3ce260..1f2d21e8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java @@ -25,10 +25,11 @@ import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.support.GenericMessage; -public class MessageUtils { +public final class MessageUtils { private static final String META_DATA_KEY_PREFIX = "__rq::m-mdata::"; + private static final String KEY_SEPARATOR = "::"; - public MessageUtils() {} + private MessageUtils() {} public static Object convertMessageToObject( RqueueMessage message, MessageConverter messageConverter) { @@ -65,7 +66,7 @@ public static Object convertMessageToObject( return null; } - public static String getMessageMetaId(String messageId) { - return META_DATA_KEY_PREFIX + messageId; + public static String getMessageMetaId(String queueName, String messageId) { + return META_DATA_KEY_PREFIX + queueName + KEY_SEPARATOR + messageId; } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/PriorityUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/PriorityUtils.java index cb723f14..6dc9fe1e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/PriorityUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/PriorityUtils.java @@ -20,8 +20,8 @@ import java.util.Map; import java.util.Set; -public class PriorityUtils { - PriorityUtils() {} +public final class PriorityUtils { + private PriorityUtils() {} public static Set getNamesFromPriority(String queueName, Map priority) { Set keys = new HashSet<>(); @@ -34,6 +34,8 @@ public static Set getNamesFromPriority(String queueName, Map nonces = (List) model.asMap().get("nonces"); + if (nonces == null) { + nonces = new ArrayList<>(); + } + nonces.add(nonce); + model.addAttribute("nonces", nonces); + model.addAttribute(name, nonce); } @GetMapping @@ -111,6 +129,7 @@ public View index(Model model, HttpServletRequest request, HttpServletResponse r } addBasicDetails(model, request); addNavData(model, null); + addNonce(model, "indexNonce"); model.addAttribute("title", "Rqueue Dashboard"); model.addAttribute("aggregatorTypes", Arrays.asList(AggregationType.values())); model.addAttribute("typeSelectors", TaskStatus.getActiveChartStatus()); @@ -153,6 +172,7 @@ public View queueDetail( rqueueQDetailService.getQueueDataStructureDetail(queueConfig); addBasicDetails(model, request); addNavData(model, NavTab.QUEUES); + addNonce(model, "queueDetailNonce"); model.addAttribute("title", "Queue: " + queueName); model.addAttribute("queueName", queueName); model.addAttribute("aggregatorTypes", Arrays.asList(AggregationType.values())); @@ -237,6 +257,7 @@ public View utility(Model model, HttpServletRequest request, HttpServletResponse } addBasicDetails(model, request); addNavData(model, NavTab.UTILITY); + addNonce(model, "utilityNonce"); model.addAttribute("title", "Utility"); model.addAttribute("supportedDataType", DataType.getEnabledDataTypes()); return rqueueViewResolver.resolveViewName("utility", Locale.ENGLISH); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java index 4a1777d8..153408e4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.web.service; +import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.models.db.MessageMetadata; import java.time.Duration; import java.util.Collection; @@ -24,11 +25,15 @@ public interface RqueueMessageMetadataService { MessageMetadata get(String id); + void delete(String id); + List findAll(Collection ids); void save(MessageMetadata messageMetadata, Duration duration); - void deleteMessage(String messageId, Duration duration); + MessageMetadata getByMessageId(String queueName, String messageId); + + void deleteMessage(String queueName, String messageId, Duration duration); - void delete(String messageMetadataId); + MessageMetadata getOrCreateMessageMetadata(RqueueMessage rqueueMessage); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java index f23e8afe..476df9ce 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java @@ -25,7 +25,7 @@ import com.github.sonus21.rqueue.models.aggregator.TasksStat; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.db.QueueStatistics; -import com.github.sonus21.rqueue.models.db.TaskStatus; +import com.github.sonus21.rqueue.models.enums.TaskStatus; import com.github.sonus21.rqueue.models.event.RqueueExecutionEvent; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.DateTimeUtils; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java index b4ab3068..d9b77d4d 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java @@ -18,7 +18,9 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.models.db.MessageStatus; import com.github.sonus21.rqueue.utils.MessageUtils; import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; @@ -26,11 +28,13 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; @Service +@Slf4j public class RqueueMessageMetadataServiceImpl implements RqueueMessageMetadataService { private final RqueueRedisTemplate template; @@ -60,11 +64,17 @@ public void save(MessageMetadata messageMetadata, Duration duration) { } @Override - public void deleteMessage(String messageId, Duration duration) { - String id = MessageUtils.getMessageMetaId(messageId); + public MessageMetadata getByMessageId(String queueName, String messageId) { + String id = MessageUtils.getMessageMetaId(queueName, messageId); + return get(id); + } + + @Override + public void deleteMessage(String queueName, String messageId, Duration duration) { + String id = MessageUtils.getMessageMetaId(queueName, messageId); MessageMetadata messageMetadata = get(id); if (messageMetadata == null) { - messageMetadata = new MessageMetadata(id, messageId); + messageMetadata = new MessageMetadata(id, MessageStatus.DELETED); } messageMetadata.setDeleted(true); messageMetadata.setDeletedOn(System.currentTimeMillis()); @@ -75,4 +85,14 @@ public void deleteMessage(String messageId, Duration duration) { public void delete(String id) { template.delete(id); } + + @Override + public MessageMetadata getOrCreateMessageMetadata(RqueueMessage rqueueMessage) { + MessageMetadata messageMetadata = + getByMessageId(rqueueMessage.getQueueName(), rqueueMessage.getId()); + if (messageMetadata == null) { + messageMetadata = new MessageMetadata(rqueueMessage, MessageStatus.ENQUEUED); + } + return messageMetadata; + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java index 3040bf9a..bd0590c9 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java @@ -47,7 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -78,8 +77,7 @@ public RqueueQDetailServiceImpl( @Override public Map>> getQueueDataStructureDetails( List queueConfig) { - return queueConfig - .parallelStream() + return queueConfig.parallelStream() .collect(Collectors.toMap(QueueConfig::getName, this::getQueueDataStructureDetail)); } @@ -172,15 +170,21 @@ private List> buildRows( if (CollectionUtils.isEmpty(rqueueMessages)) { return Collections.emptyList(); } - List ids = - rqueueMessages.stream() - .map(e -> Objects.requireNonNull(e.getValue()).getId()) - .map(MessageUtils::getMessageMetaId) - .collect(Collectors.toList()); - - List vals = rqueueMessageMetadataService.findAll(ids); - Map msgIdToDeleted = - vals.stream().collect(Collectors.toMap(MessageMetadata::getMessageId, e -> true)); + Map messageMetaIdToId = new HashMap<>(); + for (TypedTuple tuple : rqueueMessages) { + RqueueMessage rqueueMessage = tuple.getValue(); + assert rqueueMessage != null; + String messageMetaId = + MessageUtils.getMessageMetaId(rqueueMessage.getQueueName(), rqueueMessage.getId()); + messageMetaIdToId.put(messageMetaId, rqueueMessage.getId()); + } + List vals = rqueueMessageMetadataService.findAll(messageMetaIdToId.keySet()); + Map msgIdToDeleted = new HashMap<>(); + for (MessageMetadata messageMetadata : vals) { + String messageMetaId = messageMetadata.getId(); + String id = messageMetaIdToId.get(messageMetaId); + msgIdToDeleted.put(id, messageMetadata.isDeleted()); + } return rqueueMessages.stream() .map( e -> diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java index aeb4f918..f977befe 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java @@ -20,7 +20,7 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; -import com.github.sonus21.rqueue.core.QueueRegistry; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.DeadLetterQueue; import com.github.sonus21.rqueue.models.db.QueueConfig; @@ -160,7 +160,7 @@ private void createOrUpdateConfigs(List queueDetails) { @Async public void onApplicationEvent(RqueueBootstrapEvent event) { if (event.isStart()) { - List queueDetails = QueueRegistry.getActiveQueueDetails(); + List queueDetails = EndpointRegistry.getActiveQueueDetails(); if (queueDetails.isEmpty()) { return; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java index d7ed5ca9..d2cca190 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java @@ -83,7 +83,7 @@ public BooleanResponse deleteMessage(String queueName, String id) { booleanResponse.setMessage("Queue config not found!"); return booleanResponse; } - messageMetadataService.deleteMessage(id, Duration.ofDays(Constants.DAYS_IN_A_MONTH)); + messageMetadataService.deleteMessage(queueName, id, Duration.ofDays(Constants.DAYS_IN_A_MONTH)); booleanResponse.setValue(true); return booleanResponse; } @@ -167,12 +167,19 @@ public BooleanResponse deleteQueueMessages(String queueName, int remainingMessag return new BooleanResponse(false); } + private boolean shouldFetchVersionDetail() { + if (!rqueueConfig.isLatestVersionCheckEnabled()) { + return false; + } + return System.currentTimeMillis() - versionFetchTime > Constants.MILLIS_IN_A_DAY; + } + @Override @SuppressWarnings("unchecked") public Pair getLatestVersion() { - if (System.currentTimeMillis() - versionFetchTime > Constants.MILLIS_IN_A_DAY) { + if (shouldFetchVersionDetail()) { Map response = - readUrl(Constants.GITHUB_API_FOR_LATEST_RELEASE, LinkedHashMap.class); + readUrl(rqueueConfig, Constants.GITHUB_API_FOR_LATEST_RELEASE, LinkedHashMap.class); if (response != null) { String tagName = (String) response.get("tag_name"); if (tagName != null && !tagName.isEmpty()) { diff --git a/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css b/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css index 1b16f764..b5d71611 100644 --- a/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css +++ b/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css @@ -513,4 +513,9 @@ section { .delete-message-btn { padding: 0 10px; +} + +#stats_chart, #latency_chart { + width: 100%; + height: 500px; } \ No newline at end of file diff --git a/rqueue-core/src/main/resources/scripts/pop_message.lua b/rqueue-core/src/main/resources/scripts/dequeue_message.lua similarity index 100% rename from rqueue-core/src/main/resources/scripts/pop_message.lua rename to rqueue-core/src/main/resources/scripts/dequeue_message.lua diff --git a/rqueue-core/src/main/resources/scripts/add_message.lua b/rqueue-core/src/main/resources/scripts/enqueue_message.lua similarity index 100% rename from rqueue-core/src/main/resources/scripts/add_message.lua rename to rqueue-core/src/main/resources/scripts/enqueue_message.lua diff --git a/rqueue-core/src/main/resources/scripts/push_message.lua b/rqueue-core/src/main/resources/scripts/move_expired_message.lua similarity index 100% rename from rqueue-core/src/main/resources/scripts/push_message.lua rename to rqueue-core/src/main/resources/scripts/move_expired_message.lua diff --git a/rqueue-core/src/main/resources/templates/rqueue/base.html b/rqueue-core/src/main/resources/templates/rqueue/base.html index f1afb781..c4772090 100644 --- a/rqueue-core/src/main/resources/templates/rqueue/base.html +++ b/rqueue-core/src/main/resources/templates/rqueue/base.html @@ -3,14 +3,18 @@ - {{ default(title, 'Rqueue Dashboard') }} + - + + {{ default(title, 'Rqueue Dashboard') }} - + @@ -94,7 +98,8 @@

Rqueue

  • - Time: 

    {{time}} ({{timeInMilli}}Ms)

    + Time: 

    {{time}} ({{timeInMilli}}Ms)

    +
  • @@ -139,11 +144,13 @@ - - - - - + + + + {% block additional_script %}{% endblock %} diff --git a/rqueue-core/src/main/resources/templates/rqueue/index.html b/rqueue-core/src/main/resources/templates/rqueue/index.html index 9c3b35b3..97d60991 100644 --- a/rqueue-core/src/main/resources/templates/rqueue/index.html +++ b/rqueue-core/src/main/resources/templates/rqueue/index.html @@ -6,7 +6,7 @@ {% endblock %} {% block additional_script %} - -