diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/RemoteCacheManager.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/RemoteCacheManager.java index 0bb49d94f28e..fb98019a6080 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/RemoteCacheManager.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/RemoteCacheManager.java @@ -356,7 +356,7 @@ private void actualStart() { executorFactory = Util.getInstance(configuration.asyncExecutorFactory().factoryClass()); } asyncExecutorService = executorFactory.getExecutor(configuration.asyncExecutorFactory().properties()); - channelFactory.start(configuration, marshaller, asyncExecutorService, + channelFactory.start(marshaller, asyncExecutorService, listenerNotifier, marshallerRegistry); counterManager.start(channelFactory, configuration, listenerNotifier); @@ -418,7 +418,7 @@ private static void registerDefaultSchemas(SerializationContext ctx, String... c } public ChannelFactory createChannelFactory() { - return new ChannelFactory(new CodecHolder(configuration.version().getCodec())); + return new ChannelFactory(configuration, new CodecHolder(configuration.version().getCodec())); } @Override diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/AbstractConfigurationChildBuilder.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/AbstractConfigurationChildBuilder.java index 1cb0211cee3c..5beb142b88bf 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/AbstractConfigurationChildBuilder.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/AbstractConfigurationChildBuilder.java @@ -100,6 +100,11 @@ public ConfigurationBuilder dnsResolverNegativeTTL(int negativeTTL) { return builder.dnsResolverNegativeTTL(negativeTTL); } + @Override + public ConfigurationBuilder serverFailureTimeout(int timeoutInMilliseconds) { + return builder.serverFailureTimeout(timeoutInMilliseconds); + } + @Override public ConfigurationBuilder forceReturnValues(boolean forceReturnValues) { return builder.forceReturnValues(forceReturnValues); diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/Configuration.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/Configuration.java index abeaa795c2d6..7c8002be01bd 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/Configuration.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/Configuration.java @@ -38,6 +38,7 @@ import static org.infinispan.client.hotrod.impl.ConfigurationProperties.REQUEST_BALANCING_STRATEGY; import static org.infinispan.client.hotrod.impl.ConfigurationProperties.SASL_MECHANISM; import static org.infinispan.client.hotrod.impl.ConfigurationProperties.SASL_PROPERTIES_PREFIX; +import static org.infinispan.client.hotrod.impl.ConfigurationProperties.SERVER_FAILURE_TIMEOUT; import static org.infinispan.client.hotrod.impl.ConfigurationProperties.SERVER_LIST; import static org.infinispan.client.hotrod.impl.ConfigurationProperties.SNI_HOST_NAME; import static org.infinispan.client.hotrod.impl.ConfigurationProperties.SO_TIMEOUT; @@ -119,6 +120,7 @@ public class Configuration { private final int dnsResolverMinTTL; private final int dnsResolverMaxTTL; private final int dnsResolverNegativeTTL; + private final int serverFailureTimeout; public Configuration(ExecutorFactoryConfiguration asyncExecutorFactory, Supplier balancingStrategyFactory, ClassLoader classLoader, ClientIntelligence clientIntelligence, ConnectionPoolConfiguration connectionPool, int connectionTimeout, Class[] consistentHashImpl, @@ -131,7 +133,8 @@ public Configuration(ExecutorFactoryConfiguration asyncExecutorFactory, Supplier TransactionConfiguration transaction, StatisticsConfiguration statistics, Features features, List contextInitializers, Map remoteCaches, - TransportFactory transportFactory, boolean tracingPropagationEnabled) { + TransportFactory transportFactory, boolean tracingPropagationEnabled, + int serverFailureTimeout) { this.asyncExecutorFactory = asyncExecutorFactory; this.balancingStrategyFactory = balancingStrategyFactory; this.maxRetries = maxRetries; @@ -166,6 +169,7 @@ public Configuration(ExecutorFactoryConfiguration asyncExecutorFactory, Supplier this.remoteCaches = remoteCaches; this.transportFactory = transportFactory; this.tracingPropagationEnabled = tracingPropagationEnabled; + this.serverFailureTimeout = serverFailureTimeout; } public ExecutorFactoryConfiguration asyncExecutorFactory() { @@ -384,6 +388,15 @@ public boolean tracingPropagationEnabled() { return tracingPropagationEnabled; } + /** + * Controls how long a server is marked as failed in milliseconds. + * Default is 30_000 milliseconds or 30 seconds. + * @return time in milliseconds + */ + public int serverFailureTimeout() { + return serverFailureTimeout; + } + @Override public String toString() { return "Configuration [asyncExecutorFactory=" + asyncExecutorFactory + ", balancingStrategyFactory=()->" + balancingStrategyFactory.get() @@ -398,6 +411,7 @@ public String toString() { + ", remoteCaches= " + remoteCaches + ", transaction=" + transaction + ", statistics=" + statistics + + ", serverFailureTimeout=" + serverFailureTimeout + "]"; } @@ -432,6 +446,7 @@ public Properties properties() { properties.setProperty(VALUE_SIZE_ESTIMATE, valueSizeEstimate()); properties.setProperty(MAX_RETRIES, maxRetries()); properties.setProperty(STATISTICS, statistics().enabled()); + properties.setProperty(SERVER_FAILURE_TIMEOUT, serverFailureTimeout()); properties.setProperty(DNS_RESOLVER_MIN_TTL, dnsResolverMinTTL); properties.setProperty(DNS_RESOLVER_MAX_TTL, dnsResolverMaxTTL); diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationBuilder.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationBuilder.java index 3403e122e66b..d0782858ad4e 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationBuilder.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationBuilder.java @@ -84,6 +84,7 @@ public class ConfigurationBuilder implements ConfigurationChildBuilder, Builder< private boolean tcpKeepAlive = false; private int valueSizeEstimate = ConfigurationProperties.DEFAULT_VALUE_SIZE; private int maxRetries = ConfigurationProperties.DEFAULT_MAX_RETRIES; + private int serverFailureTimeout = ConfigurationProperties.DEFAULT_SERVER_FAILURE_TIMEOUT; private final NearCacheConfigurationBuilder nearCache; private final List allowListRegExs = new ArrayList<>(); private int batchSize = ConfigurationProperties.DEFAULT_BATCH_SIZE; @@ -357,6 +358,12 @@ public ConfigurationBuilder maxRetries(int maxRetries) { return this; } + @Override + public ConfigurationBuilder serverFailureTimeout(int timeoutInMilliseconds) { + this.serverFailureTimeout = timeoutInMilliseconds; + return this; + } + @Override public ConfigurationBuilder addJavaSerialAllowList(String... regEx) { this.allowListRegExs.addAll(Arrays.asList(regEx)); @@ -485,6 +492,9 @@ public ConfigurationBuilder withProperties(Properties properties) { if (typed.containsKey(ConfigurationProperties.MAX_RETRIES)) { this.maxRetries(typed.getIntProperty(ConfigurationProperties.MAX_RETRIES, maxRetries, true)); } + if (typed.containsKey(ConfigurationProperties.SERVER_FAILURE_TIMEOUT)) { + this.serverFailureTimeout(typed.getIntProperty(ConfigurationProperties.SERVER_FAILURE_TIMEOUT, serverFailureTimeout, true)); + } if (typed.containsKey(ConfigurationProperties.DNS_RESOLVER_MIN_TTL)) { this.dnsResolverMinTTL(typed.getIntProperty(ConfigurationProperties.DNS_RESOLVER_MIN_TTL, dnsResolverMinTTL, true)); } @@ -626,7 +636,7 @@ public Configuration create() { forceReturnValues, keySizeEstimate, buildMarshaller, buildMarshallerClass, protocolVersion, servers, socketTimeout, security.create(), tcpNoDelay, tcpKeepAlive, valueSizeEstimate, maxRetries, nearCache.create(), serverClusterConfigs, allowListRegExs, batchSize, transaction.create(), statistics.create(), features, - contextInitializers, remoteCaches, transportFactory, tracingPropagationEnabled); + contextInitializers, remoteCaches, transportFactory, tracingPropagationEnabled, serverFailureTimeout); } // Method that handles default marshaller - needed as a placeholder diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationChildBuilder.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationChildBuilder.java index 24750a3905a3..846eb80b28fa 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationChildBuilder.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/ConfigurationChildBuilder.java @@ -195,6 +195,17 @@ public interface ConfigurationChildBuilder { */ ConfigurationBuilder maxRetries(int maxRetries); + /** + * The time for a failed server to be cleared allowing for it to attempt to reconnect at a later point. + *

+ * If the value is less than or equal to 0 it will be disabled, meaning a failed server will not be reconnected to + * until all configured servers have failed or a topology update ({@link ClientIntelligence#TOPOLOGY_AWARE} and + * {@link ClientIntelligence#HASH_DISTRIBUTION_AWARE} only) + * @param timeoutInMilliseconds the timeout to attempt to clear a failed server in milliseconds + * @return this builder + */ + ConfigurationBuilder serverFailureTimeout(int timeoutInMilliseconds); + /** * List of regular expressions for classes that can be deserialized using standard Java deserialization * when reading data that might have been stored with a different endpoint, e.g. REST. diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/package-info.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/package-info.java index b1285e1b5e71..0df40ffa2320 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/package-info.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/configuration/package-info.java @@ -80,6 +80,12 @@ * The {@link org.infinispan.client.hotrod.configuration.ConfigurationBuilder#connectionTimeout(int) timeout} for connections * * + * infinispan.client.hotrod.server_failure_timeout + * Integer + * 30000 + * The {@link org.infinispan.client.hotrod.configuration.ConfigurationBuilder#serverFailureTimeout(int) timeout} for a failed server when it is retried + * + * * infinispan.client.hotrod.max_retries * Integer * 2 diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/ConfigurationProperties.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/ConfigurationProperties.java index 8b0b258a43b0..4d05790ec97e 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/ConfigurationProperties.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/ConfigurationProperties.java @@ -53,6 +53,7 @@ public class ConfigurationProperties { public static final String CONNECT_TIMEOUT = ICH + "connect_timeout"; public static final String PROTOCOL_VERSION = ICH + "protocol_version"; public static final String TRANSPORT_FACTORY = ICH + "transport_factory"; + public static final String SERVER_FAILURE_TIMEOUT = ICH + "server_failure_timeout"; // Encryption properties public static final String USE_SSL = ICH + "use_ssl"; public static final String KEY_STORE_FILE_NAME = ICH + "key_store_file_name"; @@ -155,6 +156,7 @@ public class ConfigurationProperties { public static final int DEFAULT_MAX_WAIT = -1; public static final int DEFAULT_MIN_IDLE = -1; public static final boolean DEFAULT_TRACING_PROPAGATION_ENABLED = true; + public static final int DEFAULT_SERVER_FAILURE_TIMEOUT = 30_000; private final TypedProperties props; diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/transport/netty/ChannelFactory.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/transport/netty/ChannelFactory.java index 53da59957490..5eb9f21f8216 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/transport/netty/ChannelFactory.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/transport/netty/ChannelFactory.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -62,6 +63,8 @@ import org.infinispan.commons.util.ProcessorInfo; import org.infinispan.commons.util.SslContextFactory; +import com.github.benmanes.caffeine.cache.Caffeine; + import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelOption; @@ -94,7 +97,6 @@ public class ChannelFactory { private EventLoopGroup eventLoopGroup; private ExecutorService executorService; private OperationsFactory operationsFactory; - private Configuration configuration; private int maxRetries; private Marshaller marshaller; private ClientListenerNotifier listenerNotifier; @@ -110,23 +112,30 @@ public class ChannelFactory { private CompletableFuture clusterSwitchStage; // Servers for which the last connection attempt failed and which have no established connections @GuardedBy("lock") - private final Set failedServers = new HashSet<>(); + private final Set failedServers; + private final Configuration configuration; private final CodecHolder codecHolder; private AddressResolverGroup dnsResolver; private SslContext sslContext; private FileWatcher watcher; - public ChannelFactory(CodecHolder codecHolder) { + public ChannelFactory(Configuration configuration, CodecHolder codecHolder) { + this.configuration = configuration; this.codecHolder = codecHolder; + + this.failedServers = configuration.serverFailureTimeout() > 0 ? + Collections.newSetFromMap(Caffeine.newBuilder() + .expireAfterWrite(configuration.serverFailureTimeout(), TimeUnit.MILLISECONDS) + .build().asMap()) + : new HashSet<>(); } - public void start(Configuration configuration, Marshaller marshaller, ExecutorService executorService, + public void start(Marshaller marshaller, ExecutorService executorService, ClientListenerNotifier listenerNotifier, MarshallerRegistry marshallerRegistry) { this.marshallerRegistry = marshallerRegistry; lock.writeLock().lock(); try { this.marshaller = marshaller; - this.configuration = configuration; this.executorService = executorService; this.listenerNotifier = listenerNotifier; int asyncThreads = maxAsyncThreads(executorService, configuration); @@ -881,6 +890,10 @@ public Configuration getConfiguration() { return configuration; } + public Set getFailedServers() { + return failedServers; + } + public long getRetries() { return totalRetries.longValue(); } diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/BasicClientIntelligenceTest.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/BasicClientIntelligenceTest.java new file mode 100644 index 000000000000..9457175de2b2 --- /dev/null +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/BasicClientIntelligenceTest.java @@ -0,0 +1,92 @@ +package org.infinispan.client.hotrod; + +import static org.infinispan.client.hotrod.test.HotRodClientTestingUtil.withRemoteCacheManager; +import static org.infinispan.server.hotrod.test.HotRodTestingUtil.hotRodCacheConfiguration; +import static org.testng.AssertJUnit.assertFalse; + +import org.infinispan.client.hotrod.configuration.ClientIntelligence; +import org.infinispan.client.hotrod.exceptions.TransportException; +import org.infinispan.client.hotrod.impl.transport.netty.ChannelFactory; +import org.infinispan.client.hotrod.test.HotRodClientTestingUtil; +import org.infinispan.client.hotrod.test.InternalRemoteCacheManager; +import org.infinispan.client.hotrod.test.MultiHotRodServersTest; +import org.infinispan.client.hotrod.test.RemoteCacheManagerCallable; +import org.infinispan.commons.test.Exceptions; +import org.infinispan.configuration.cache.CacheMode; +import org.infinispan.configuration.cache.ConfigurationBuilder; +import org.testng.annotations.Test; + +@Test(groups = "functional", testName = "client.hotrod.BasicClientIntelligenceTest") +public class BasicClientIntelligenceTest extends MultiHotRodServersTest { + private final ConfigurationBuilder builder = hotRodCacheConfiguration( + getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, false)); + @Override + protected void createCacheManagers() throws Throwable { + createHotRodServersWithoutClients(2, builder); + } + + public void testOneServerDiedAndComesBack() { + int initialPort = server(1).getPort(); + + org.infinispan.client.hotrod.configuration.ConfigurationBuilder clientBuilder = + new org.infinispan.client.hotrod.configuration.ConfigurationBuilder(); + // Retry after every half second to make test faster + clientBuilder.serverFailureTimeout(500); + clientBuilder.addServers(HotRodClientTestingUtil.getServersString(server(0), server(1))); + clientBuilder.clientIntelligence(ClientIntelligence.BASIC); + + withRemoteCacheManager(new RemoteCacheManagerCallable( + new InternalRemoteCacheManager(clientBuilder.build())) { + @Override + public void call() { + RemoteCache cache = rcm.getCache(); + ChannelFactory cf = rcm.getChannelFactory(); + assertFalse(cache.containsKey("k")); + killServer(1); + eventuallyEquals(1, () -> { + assertFalse(cache.containsKey("k")); + return cf.getFailedServers().size(); + }); + + addHotRodServer(builder, initialPort); + + eventuallyEquals(0, () -> { + assertFalse(cache.containsKey("k")); + return cf.getFailedServers().size(); + }); + } + }); + } + + public void testBasicHasSingleServerThatDied() { + int initialPort = server(1).getPort(); + + org.infinispan.client.hotrod.configuration.ConfigurationBuilder clientBuilder = + new org.infinispan.client.hotrod.configuration.ConfigurationBuilder(); + clientBuilder.serverFailureTimeout(500); + clientBuilder.addServers(HotRodClientTestingUtil.getServersString(server(1))); + clientBuilder.clientIntelligence(ClientIntelligence.BASIC); + + withRemoteCacheManager(new RemoteCacheManagerCallable( + new InternalRemoteCacheManager(clientBuilder.build())) { + @Override + public void call() { + RemoteCache cache = rcm.getCache(); + ChannelFactory cf = rcm.getChannelFactory(); + assertFalse(cache.containsKey("k")); + killServer(1); + for (int i = 0; i < 10; i++) { + Exceptions.expectException(TransportException.class, () -> cache.containsKey("k")); + } + eventuallyEquals(1, () -> cf.getFailedServers().size()); + + addHotRodServer(builder, initialPort); + + eventuallyEquals(0, () -> { + assertFalse(cache.containsKey("k")); + return cf.getFailedServers().size(); + }); + } + }); + } +} diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/hash/ConsistentHashTest.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/hash/ConsistentHashTest.java index 849715be0395..463771f54b5e 100644 --- a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/hash/ConsistentHashTest.java +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/hash/ConsistentHashTest.java @@ -120,7 +120,7 @@ private static class ControlledChannelFactory extends ChannelFactory { private BiConsumer onFetch; public ControlledChannelFactory(Configuration cfg) { - super(new CodecHolder(cfg.version().getCodec())); + super(cfg, new CodecHolder(cfg.version().getCodec())); } public void useOnFetch(BiConsumer onFetch) { diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/CloseBeforeEnqueuingTest.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/CloseBeforeEnqueuingTest.java index 2f9c48cf9c73..a086b45fec02 100644 --- a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/CloseBeforeEnqueuingTest.java +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/CloseBeforeEnqueuingTest.java @@ -257,7 +257,7 @@ private static class CustomChannelFactory extends ChannelFactory { private Supplier executeInstead; public CustomChannelFactory(Configuration cfg) { - super(new CodecHolder(cfg.version().getCodec())); + super(cfg, new CodecHolder(cfg.version().getCodec())); this.configuration = cfg; this.executeInstead = null; } diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/TestChannelFactory.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/TestChannelFactory.java index 52d80865dbba..55c4b40ae757 100644 --- a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/TestChannelFactory.java +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/impl/transport/netty/TestChannelFactory.java @@ -2,6 +2,7 @@ import java.net.SocketAddress; +import org.infinispan.client.hotrod.configuration.Configuration; import org.infinispan.client.hotrod.impl.protocol.CodecHolder; import io.netty.bootstrap.Bootstrap; @@ -9,8 +10,8 @@ import io.netty.handler.codec.FixedLengthFrameDecoder; public class TestChannelFactory extends ChannelFactory { - public TestChannelFactory(CodecHolder codecHolder) { - super(codecHolder); + public TestChannelFactory(Configuration configuration, CodecHolder codecHolder) { + super(configuration, codecHolder); } @Override diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/InternalRemoteCacheManager.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/InternalRemoteCacheManager.java index 61ce9ceb3f21..17dbfe8b484a 100644 --- a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/InternalRemoteCacheManager.java +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/InternalRemoteCacheManager.java @@ -54,7 +54,7 @@ public ChannelFactory getChannelFactory() { @Override public ChannelFactory createChannelFactory() { if (customChannelFactory != null) return customChannelFactory; - if (testReplay) return new TestChannelFactory(new CodecHolder(getConfiguration().version().getCodec())); + if (testReplay) return new TestChannelFactory(getConfiguration(), new CodecHolder(getConfiguration().version().getCodec())); return super.createChannelFactory(); } } diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/MultiHotRodServersTest.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/MultiHotRodServersTest.java index 084209adbe81..1774affd8980 100644 --- a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/MultiHotRodServersTest.java +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/test/MultiHotRodServersTest.java @@ -37,6 +37,14 @@ public abstract class MultiHotRodServersTest extends MultipleCacheManagersTest { protected boolean testReplay = true; protected void createHotRodServers(int num, ConfigurationBuilder defaultBuilder) { + createHotRodServersWithoutClients(num, defaultBuilder); + + for (int i = 0; i < num; i++) { + clients.add(createClient(i)); + } + } + + protected void createHotRodServersWithoutClients(int num, ConfigurationBuilder defaultBuilder) { // Start Hot Rod servers for (int i = 0; i < num; i++) addHotRodServer(defaultBuilder); // Verify that default caches should be started @@ -48,10 +56,6 @@ protected void createHotRodServers(int num, ConfigurationBuilder defaultBuilder) blockUntilCacheStatusAchieved( manager(i).getCache(), ComponentStatus.RUNNING, 10000); } - - for (int i = 0; i < num; i++) { - clients.add(createClient(i)); - } } protected RemoteCacheManager createClient(int i) { diff --git a/client/hotrod/src/main/java/org/infinispan/hotrod/configuration/package-info.java b/client/hotrod/src/main/java/org/infinispan/hotrod/configuration/package-info.java index 70f7a0f0d71c..5a666b1dd773 100644 --- a/client/hotrod/src/main/java/org/infinispan/hotrod/configuration/package-info.java +++ b/client/hotrod/src/main/java/org/infinispan/hotrod/configuration/package-info.java @@ -80,6 +80,12 @@ * The {@link org.infinispan.hotrod.configuration.HotRodConfigurationBuilder#connectionTimeout(int) timeout} for connections * * + * infinispan.client.hotrod.server_failure_timeout + * Integer + * 30000 + * The {@link org.infinispan.client.hotrod.configuration.ConfigurationBuilder#serverFailureTimeout(int) timeout} for a failed server when it is retried + * + * * infinispan.client.hotrod.max_retries * Integer * 10 diff --git a/client/hotrod/src/main/java/org/infinispan/hotrod/impl/ConfigurationProperties.java b/client/hotrod/src/main/java/org/infinispan/hotrod/impl/ConfigurationProperties.java index e04125d532eb..77d5afb00600 100644 --- a/client/hotrod/src/main/java/org/infinispan/hotrod/impl/ConfigurationProperties.java +++ b/client/hotrod/src/main/java/org/infinispan/hotrod/impl/ConfigurationProperties.java @@ -42,6 +42,7 @@ public class ConfigurationProperties { public static final String CONNECT_TIMEOUT = ICH + "connect_timeout"; public static final String PROTOCOL_VERSION = ICH + "protocol_version"; public static final String TRANSPORT_FACTORY = ICH + "transport_factory"; + public static final String SERVER_FAILURE_TIMEOUT = ICH + "server_failure_timeout"; // Encryption properties public static final String USE_SSL = ICH + "use_ssl"; public static final String KEY_STORE_FILE_NAME = ICH + "key_store_file_name"; @@ -128,6 +129,7 @@ public class ConfigurationProperties { public static final int DEFAULT_MAX_WAIT = -1; public static final int DEFAULT_MIN_IDLE = -1; public static final boolean DEFAULT_TRACING_PROPAGATION_ENABLED = true; + public static final int DEFAULT_SERVER_FAILURE_TIMEOUT = 30_000; private final TypedProperties props; diff --git a/documentation/src/main/asciidoc/topics/code_examples/HotRodClientFailedServerTimeout.java b/documentation/src/main/asciidoc/topics/code_examples/HotRodClientFailedServerTimeout.java new file mode 100644 index 000000000000..5968fb3dc44a --- /dev/null +++ b/documentation/src/main/asciidoc/topics/code_examples/HotRodClientFailedServerTimeout.java @@ -0,0 +1,2 @@ +ConfigurationBuilder builder = new ConfigurationBuilder(); +builder.serverFailureTimeout(5000).clusterClientIntelligence(ClientIntelligence.BASIC); diff --git a/documentation/src/main/asciidoc/topics/con_hotrod_client_intelligence.adoc b/documentation/src/main/asciidoc/topics/con_hotrod_client_intelligence.adoc index 8e2439b66f0a..a376e6f34d5c 100644 --- a/documentation/src/main/asciidoc/topics/con_hotrod_client_intelligence.adoc +++ b/documentation/src/main/asciidoc/topics/con_hotrod_client_intelligence.adoc @@ -58,6 +58,31 @@ include::code_examples/HotRodClientIntelligenceCluster.java[] include::properties/hotrod-client-intelligence-cluster.properties[] ---- + +[discrete] +== Failed Server Timeout + +If a server does not report the topology as BASIC, or the client is unable to connect to a server due to network issues, the client will mark the server as failed. +A client does not attempt to connect to a server marked as failed until the client receives an updated topology. +Because BASIC topology never sends an update, the client will not re-attempt connection. + +To avoid such a situation, you can use the `serverFailureTimeout` setting that clears the failed server status after a defined period of time. +{brandname} will try to reconnect to the server after the defined timeout. If the server is still unreachable, it is marked as failed again and the connection will be re-attempted after the defined timeout. +You can disabled reconnection attempts by setting the `serverFailureTimeout` value to `-1`. + +.ConfigurationBuilder +[source,java,options="nowrap",subs=attributes+,role="primary"] +---- +include::code_examples/HotRodClientFailedServerTimeout.java[] +---- + +.hotrod-client.properties +[source,options="nowrap",subs=attributes+,role="secondary"] +---- +include::properties/hotrod-client-failed-server-timeout.properties[] +---- + [role="_additional-resources"] .Additional resources * link:{javadocroot}/org/infinispan/client/hotrod/configuration/ClientIntelligence.html[org.infinispan.client.hotrod.configuration.ClientIntelligence] + diff --git a/documentation/src/main/asciidoc/topics/properties/hotrod-client-failed-server-timeout.properties b/documentation/src/main/asciidoc/topics/properties/hotrod-client-failed-server-timeout.properties new file mode 100644 index 000000000000..8447854994dd --- /dev/null +++ b/documentation/src/main/asciidoc/topics/properties/hotrod-client-failed-server-timeout.properties @@ -0,0 +1,2 @@ +infinispan.client.hotrod.server_failure_timeout=5000 +infinispan.client.hotrod.client_intelligence=BASIC