Merge pull request #5 from atlanhq/redisleak

PLT-541: make redis client singleton
atlanhq · Jan 3, 2024 · 9d45e3e · 9d45e3e
2 parents e7209e1 + 6df49ad
commit 9d45e3e
Show file tree

Hide file tree

Showing 30 changed files with 102 additions and 63 deletions.
diff --git a/docs/configs/janusgraph-cfg.md b/docs/configs/janusgraph-cfg.md
@@ -27,6 +27,8 @@ Configuration options that modify JanusGraph's caching behavior
 | cache.db-cache-time | Default expiration time, in milliseconds, for entries in the database-level cache. Entries are evicted when they reach this age even if the cache has room to spare. Set to 0 to disable expiration (cache entries live forever or until memory pressure triggers eviction when set to 0). | Long | 10000 | GLOBAL_OFFLINE |
 | cache.redis-cache-connectTimeout | Timeout during connecting to any Redis server. | Integer | 1000 | MASKABLE |
 | cache.redis-cache-keepAlive | Enables TCP keepAlive for connection. | Boolean | true | MASKABLE |
+| cache.redis-cache-lease-ms | If the lock is not available then the current thread becomes disabled for thread scheduling purposes and lies dormant until the lock has been acquired. If the lock is acquired, it is held until unlock is invoked, or until leaseTime milliseconds have passed since the lock was granted - whichever comes first | Integer | 2000 | MASKABLE |
+| cache.redis-cache-lock-wait-ms | The maximum time (in millisec) to aquire the lock. | Integer | 1000 | MASKABLE |
 | cache.redis-cache-lock-watchdog-ms | This prevents against infinity locked locks due to Redisson client crush or any other reason when lock can't be released in proper way. | Long | 600000 | MASKABLE |
 | cache.redis-cache-mastername | Master server name used by Redis Sentinel servers and master change monitoring task. | String | mymaster | MASKABLE |
 | cache.redis-cache-password | Password for Redis authentication. | String | password | MASKABLE |

diff --git a/janusgraph-all/pom.xml b/janusgraph-all/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-all</artifactId>

diff --git a/janusgraph-backend-testutils/pom.xml b/janusgraph-backend-testutils/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-backend-testutils</artifactId>

diff --git a/janusgraph-berkeleyje/pom.xml b/janusgraph-berkeleyje/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-berkeleyje</artifactId>

diff --git a/janusgraph-bigtable/pom.xml b/janusgraph-bigtable/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-bigtable</artifactId>

diff --git a/janusgraph-core/pom.xml b/janusgraph-core/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-core</artifactId>

diff --git a/...c/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/ExpirationKCVSRedisCache.java b/...c/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/ExpirationKCVSRedisCache.java
@@ -42,6 +42,8 @@
 import java.util.logging.Logger;
 
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.CACHE_KEYSPACE_PREFIX;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_LOCK_LEASE_MS;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_LOCK_WAIT_MS;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_MAX_CACHE_SIZE;
 
 /**
@@ -56,10 +58,12 @@ public class ExpirationKCVSRedisCache extends KCVSCache {
     private RMapCache<StaticBuffer, ArrayList<KeySliceQuery>> redisIndexKeys;
     private static Logger logger = Logger.getLogger("janusgraph-redis-logger");
     private static FSTConfiguration fastConf = FSTConfiguration.createDefaultConfiguration();
+    private Configuration configuration;
 
     public ExpirationKCVSRedisCache(final KeyColumnValueStore store, String metricsName, final long cacheTimeMS,
                                     final long invalidationGracePeriodMS, Configuration configuration) {
         super(store, metricsName);
+        this.configuration = configuration;
         Preconditions.checkArgument(cacheTimeMS > 0, "Cache expiration must be positive: %s", cacheTimeMS);
         Preconditions.checkArgument(System.currentTimeMillis() + 1000L * 3600 * 24 * 365 * 100 + cacheTimeMS > 0, "Cache expiration time too large, overflow may occur: %s", cacheTimeMS);
         this.cacheTimeMS = cacheTimeMS;
@@ -101,7 +105,7 @@ private EntryList get(KeySliceQuery query, Callable<EntryList> valueLoader) thro
                     redisCache.fastPutAsync(query, fastConf.asByteArray(entries), this.cacheTimeMS, TimeUnit.MILLISECONDS);
                     RLock lock = redisIndexKeys.getLock(query.getKey());
                     try {
-                        lock.tryLock(1, 3, TimeUnit.SECONDS);
+                        lock.tryLock(this.configuration.get(REDIS_CACHE_LOCK_WAIT_MS), this.configuration.get(REDIS_CACHE_LOCK_LEASE_MS), TimeUnit.MILLISECONDS);
                         ArrayList<KeySliceQuery> queryList = redisIndexKeys.get(query.getKey());
                         if (queryList == null)
                             queryList = new ArrayList<>();
@@ -154,7 +158,7 @@ public Map<StaticBuffer, EntryList> getSlice(final List<StaticBuffer> keys, fina
                         redisCache.fastPutAsync(ksqs[i], fastConf.asByteArray(subresult), this.cacheTimeMS, TimeUnit.MILLISECONDS);
                         RLock lock = redisIndexKeys.getLock(ksqs[i].getKey());
                         try {
-                            lock.tryLock(3, 10, TimeUnit.SECONDS);
+                            lock.tryLock(this.configuration.get(REDIS_CACHE_LOCK_WAIT_MS), this.configuration.get(REDIS_CACHE_LOCK_LEASE_MS), TimeUnit.MILLISECONDS);
                             ArrayList<KeySliceQuery> queryList = redisIndexKeys.get(ksqs[i].getKey());
                             if (queryList == null)
                                 queryList = new ArrayList<>();

diff --git a/...aph-core/src/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/RedissonCache.java b/...aph-core/src/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/RedissonCache.java
@@ -25,12 +25,24 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.Arrays;
+import java.util.Objects;
 
-import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.*;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_CONNECTION_TIME_OUT;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_KEEP_ALIVE;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_LOCK_WATCHDOG_TIMEOUT_MS;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_MASTER_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_PASSWORD;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_SENTINEL_URLS;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_SERVER_MODE;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_SERVER_URL;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_USERNAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CLIENT_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_DATABASE_ID;
 
 public class RedissonCache {
 
     private static final Logger log = LoggerFactory.getLogger(RedissonCache.class);
+    private static RedissonClient client;
     private static final String REDIS_URL_PREFIX = "redis://";
     private static final String COMMA = ",";
     private static final String SENTINEL = "sentinel";
@@ -41,41 +53,51 @@ public class RedissonCache {
     private static long watchdogTimeoutInMS;
 
     public static RedissonClient getRedissonClient(Configuration configuration) {
-        redisServerMode = configuration.get(REDIS_CACHE_SERVER_MODE);
-        connectTimeout = configuration.get(REDIS_CACHE_CONNECTION_TIME_OUT);
-        keepAlive = configuration.get(REDIS_CACHE_KEEP_ALIVE);
-        watchdogTimeoutInMS = configuration.get(REDIS_CACHE_LOCK_WATCHDOG_TIMEOUT_MS);
-        log.info("Creating connection for redis:{}", redisServerMode);
-        Config config = new Config();
-        switch (redisServerMode) {
-            case SENTINEL:
-                config.setLockWatchdogTimeout(watchdogTimeoutInMS)
-                    .useSentinelServers()
-                    .setDatabase(configuration.get(REDIS_DATABASE_ID))
-                    .setClientName(configuration.get(REDIS_CLIENT_NAME))
-                    .setReadMode(ReadMode.MASTER_SLAVE)
-                    .setCheckSentinelsList(false)
-                    .setConnectTimeout(connectTimeout)
-                    .setKeepAlive(keepAlive)
-                    .setMasterName(configuration.get(REDIS_CACHE_MASTER_NAME))
-                    .addSentinelAddress(formatUrls(configuration.get(REDIS_CACHE_SENTINEL_URLS).split(COMMA)))
-                    .setUsername(configuration.get(REDIS_CACHE_USERNAME))
-                    .setPassword(configuration.get(REDIS_CACHE_PASSWORD));
-                break;
-            case STANDALONE:
-                config.setLockWatchdogTimeout(watchdogTimeoutInMS)
-                    .useSingleServer()
-                    .setClientName(configuration.get(REDIS_CLIENT_NAME))
-                    .setAddress(formatUrls(configuration.get(REDIS_CACHE_SERVER_URL).split(COMMA))[0])
-                    .setConnectTimeout(connectTimeout)
-                    .setKeepAlive(keepAlive)
-                    .setUsername(configuration.get(REDIS_CACHE_USERNAME))
-                    .setPassword(configuration.get(REDIS_CACHE_PASSWORD));
-                break;
-            default:
-                throw new JanusGraphConfigurationException("Invalid redis server mode");
+        synchronized (RedissonCache.class) {
+            if (Objects.isNull(client)) {
+                redisServerMode = configuration.get(REDIS_CACHE_SERVER_MODE);
+                connectTimeout = configuration.get(REDIS_CACHE_CONNECTION_TIME_OUT);
+                keepAlive = configuration.get(REDIS_CACHE_KEEP_ALIVE);
+                watchdogTimeoutInMS = configuration.get(REDIS_CACHE_LOCK_WATCHDOG_TIMEOUT_MS);
+                log.info("Creating connection for redis:{}", redisServerMode);
+                Config config = new Config();
+                switch (redisServerMode) {
+                    case SENTINEL:
+                        config.setLockWatchdogTimeout(watchdogTimeoutInMS)
+                            .useSentinelServers()
+                            .setDatabase(configuration.get(REDIS_DATABASE_ID))
+                            .setClientName(String.join("-",configuration.get(REDIS_CLIENT_NAME),"janusgraph"))
+                            .setReadMode(ReadMode.MASTER_SLAVE)
+                            .setCheckSentinelsList(false)
+                            .setConnectTimeout(connectTimeout)
+                            .setIdleConnectionTimeout(5_000)
+                            .setMasterConnectionMinimumIdleSize(10)
+                            .setMasterConnectionPoolSize(20)
+                            .setSlaveConnectionMinimumIdleSize(10)
+                            .setSlaveConnectionPoolSize(20)
+                            .setKeepAlive(keepAlive)
+                            .setMasterName(configuration.get(REDIS_CACHE_MASTER_NAME))
+                            .addSentinelAddress(formatUrls(configuration.get(REDIS_CACHE_SENTINEL_URLS).split(COMMA)))
+                            .setUsername(configuration.get(REDIS_CACHE_USERNAME))
+                            .setPassword(configuration.get(REDIS_CACHE_PASSWORD));
+                        break;
+                    case STANDALONE:
+                        config.setLockWatchdogTimeout(watchdogTimeoutInMS)
+                            .useSingleServer()
+                            .setClientName(configuration.get(REDIS_CLIENT_NAME))
+                            .setAddress(formatUrls(configuration.get(REDIS_CACHE_SERVER_URL).split(COMMA))[0])
+                            .setConnectTimeout(connectTimeout)
+                            .setKeepAlive(keepAlive)
+                            .setUsername(configuration.get(REDIS_CACHE_USERNAME))
+                            .setPassword(configuration.get(REDIS_CACHE_PASSWORD));
+                        break;
+                    default:
+                        throw new JanusGraphConfigurationException("Invalid redis server mode");
+                }
+                client = Redisson.create(config);
+            }
         }
-        return Redisson.create(config);
+        return client;
     }
 
     private static String[] formatUrls(String[] urls) throws IllegalArgumentException {

diff --git a/...h-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java b/...h-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java
@@ -418,6 +418,17 @@ public boolean apply(@Nullable String s) {
         "Maximum cache (map) size in redis, set 0 to unbound, default value is 100. Keys are evicted based on LFU mode.",
         ConfigOption.Type.MASKABLE, 100);
 
+    public static final ConfigOption<Integer> REDIS_CACHE_LOCK_WAIT_MS = new ConfigOption<>(CACHE_NS,"redis-cache-lock-wait-ms",
+        "The maximum time (in millisec) to aquire the lock.",
+        ConfigOption.Type.MASKABLE, 1000);
+
+
+    public static final ConfigOption<Integer> REDIS_CACHE_LOCK_LEASE_MS = new ConfigOption<>(CACHE_NS,"redis-cache-lease-ms",
+        "If the lock is not available then the current thread becomes disabled for thread scheduling purposes " +
+            "and lies dormant until the lock has been acquired. If the lock is acquired, it is held until unlock is invoked, " +
+            "or until leaseTime milliseconds have passed since the lock was granted - whichever comes first",
+        ConfigOption.Type.MASKABLE, 2000);
+
     /**
      * The size of the database level cache.
      * If this value is between 0.0 (strictly bigger) and 1.0 (strictly smaller), then it is interpreted as a

diff --git a/janusgraph-cql/pom.xml b/janusgraph-cql/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
     </parent>
 
     <artifactId>janusgraph-cql</artifactId>

diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <packaging>pom</packaging>

diff --git a/janusgraph-doc/pom.xml b/janusgraph-doc/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <packaging>pom</packaging>

diff --git a/janusgraph-driver/pom.xml b/janusgraph-driver/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-driver</artifactId>

diff --git a/janusgraph-es/pom.xml b/janusgraph-es/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-es</artifactId>

diff --git a/janusgraph-examples/example-berkeleyje/pom.xml b/janusgraph-examples/example-berkeleyje/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-berkeleyje</artifactId>

diff --git a/janusgraph-examples/example-common/pom.xml b/janusgraph-examples/example-common/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-common</artifactId>

diff --git a/janusgraph-examples/example-cql/pom.xml b/janusgraph-examples/example-cql/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-cql</artifactId>

diff --git a/janusgraph-examples/example-hbase/pom.xml b/janusgraph-examples/example-hbase/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-hbase</artifactId>

diff --git a/janusgraph-examples/example-remotegraph/pom.xml b/janusgraph-examples/example-remotegraph/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-remotegraph</artifactId>

diff --git a/janusgraph-examples/example-tinkergraph/pom.xml b/janusgraph-examples/example-tinkergraph/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-tinkergraph</artifactId>

diff --git a/janusgraph-examples/pom.xml b/janusgraph-examples/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-examples</artifactId>

diff --git a/janusgraph-grpc/pom.xml b/janusgraph-grpc/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
     </parent>
     <artifactId>janusgraph-grpc</artifactId>
     <name>JanusGraph-gRPC: gRPC Components for JanusGraph</name>

diff --git a/janusgraph-hadoop/pom.xml b/janusgraph-hadoop/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-hadoop</artifactId>

diff --git a/janusgraph-hbase/pom.xml b/janusgraph-hbase/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
     </parent>
     <artifactId>janusgraph-hbase</artifactId>
     <name>JanusGraph-HBase: Distributed Graph Database</name>

diff --git a/janusgraph-inmemory/pom.xml b/janusgraph-inmemory/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-inmemory</artifactId>

diff --git a/janusgraph-lucene/pom.xml b/janusgraph-lucene/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-lucene</artifactId>

diff --git a/janusgraph-server/pom.xml b/janusgraph-server/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
     </parent>
     <artifactId>janusgraph-server</artifactId>
     <name>JanusGraph-Server: Server Components for JanusGraph</name>

diff --git a/janusgraph-solr/pom.xml b/janusgraph-solr/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.02</version>
+        <version>0.6.03</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-solr</artifactId>