Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make locks dynamic based on TTL of lock table #587

Merged
merged 14 commits into from
Oct 20, 2023
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.datastax.oss.driver.api.core.CqlSession;
import com.datastax.oss.driver.api.core.metadata.Node;
import com.ericsson.bss.cassandra.ecchronos.application.config.Config;
import com.ericsson.bss.cassandra.ecchronos.application.config.lockfactory.CasLockFactoryConfig;
import com.ericsson.bss.cassandra.ecchronos.connection.JmxConnectionProvider;
import com.ericsson.bss.cassandra.ecchronos.connection.NativeConnectionProvider;
import com.ericsson.bss.cassandra.ecchronos.connection.StatementDecorator;
Expand Down Expand Up @@ -75,11 +76,15 @@ public ECChronosInternals(final Config configuration,
.withJmxProxyFactory(myJmxProxyFactory)
.build();

CasLockFactoryConfig casLockFactoryConfig = configuration.getLockFactory().getCasLockFactoryConfig();
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved
myLockFactory = CASLockFactory.builder()
.withNativeConnectionProvider(nativeConnectionProvider)
.withHostStates(myHostStatesImpl)
.withStatementDecorator(statementDecorator)
.withKeyspaceName(configuration.getLockFactory().getCasLockFactoryConfig().getKeyspaceName())
.withKeyspaceName(casLockFactoryConfig.getKeyspaceName())
.withLockTimeInSeconds(casLockFactoryConfig.getLockTimeInSeconds())
.withLockUpdateTimeInSeconds(casLockFactoryConfig.getLockUpdateTimeInSeconds())
.withCacheExpiryInSeconds(casLockFactoryConfig.getExpiryTimeInSeconds())
.build();

Node node = nativeConnectionProvider.getLocalNode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,47 @@
public class CasLockFactoryConfig
{
private String myKeyspaceName = "ecchronos";
private long myLockTimeInSeconds = 600L;
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved
private long myLockUpdateTimeInSeconds = 60L;
private long myExpiryTimeInSeconds = 30L;
@JsonProperty ("lock_time_in_seconds")
public final long getLockTimeInSeconds()
{
return myLockTimeInSeconds;
}

@JsonProperty ("lock_time_in_seconds")
public final void setLockTimeInSeconds(final long lockTimeInSeconds)
{
myLockTimeInSeconds = lockTimeInSeconds;
}

@JsonProperty ("lock_update_time_in_seconds")
public final long getLockUpdateTimeInSeconds()
{
return myLockUpdateTimeInSeconds;
}

@JsonProperty("keyspace")
@JsonProperty ("lock_update_time_in_seconds")
public final void setLockUpdateTimeInSeconds(final long lockUpdateTimeInSeconds)
{
myLockTimeInSeconds = lockUpdateTimeInSeconds;
}
@JsonProperty ("cache_expiry_time_in_second")
public long getExpiryTimeInSeconds() {
return myExpiryTimeInSeconds;
}
@JsonProperty ("cache_expiry_time_in_second")
public void setExpiryTimeInSeconds(long expiryTimeInSeconds) {
myExpiryTimeInSeconds = expiryTimeInSeconds;
}
@JsonProperty ("keyspace")
public final String getKeyspaceName()
{
return myKeyspaceName;
}

@JsonProperty("keyspace")
@JsonProperty ("keyspace")
masokol marked this conversation as resolved.
Show resolved Hide resolved
public final void setKeyspaceName(final String keyspaceName)
{
myKeyspaceName = keyspaceName;
Expand Down
12 changes: 12 additions & 0 deletions application/src/main/resources/ecc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,18 @@ lock_factory:
## The keyspace used for the CAS lock factory tables.
##
keyspace: ecchronos
##
## The duration for which the lock is held, in seconds
##
lock_time_in_seconds: 600
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved
##
## The interval at which the lock is updated, in seconds
##
lock_update_time_in_seconds: 60
##
## The time after which the lock cache expires, in seconds", in seconds
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved
##
cache_expiry_time_in_second: 30
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved

run_policy:
time_based:
Expand Down
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
public class CASLockFactoryService implements LockFactory
{
private static final String DEFAULT_KEYSPACE_NAME = "ecchronos";
SajidRiaz138 marked this conversation as resolved.
Show resolved Hide resolved
private static final long DEFAULT_LOCK_TIME_IN_SECONDS = 600L;
private static final long DEFAULT_UPDATE_TIME_IN_SECONDS = 60L;

@Reference(service = NativeConnectionProvider.class,
cardinality = ReferenceCardinality.MANDATORY,
Expand All @@ -64,6 +66,8 @@ public final synchronized void activate(final Configuration configuration)
.withHostStates(myHostStates)
.withStatementDecorator(myStatementDecorator)
.withKeyspaceName(configuration.keyspaceName())
.withLockTimeInSeconds(configuration.lockTimeInSeconds())
.withLockUpdateTimeInSeconds(configuration.lockUpdateTimeInSeconds())
.build();
}

Expand Down Expand Up @@ -101,5 +105,13 @@ public final boolean sufficientNodesForLocking(final String dataCenter, final St
@AttributeDefinition(name = "The lock factory keyspace to use",
description = "The name of the keyspace containing the lock factory tables")
String keyspaceName() default DEFAULT_KEYSPACE_NAME;
@AttributeDefinition(name = "Lock Time",
description = "The duration for which the lock is held, in seconds")
long lockTimeInSeconds() default DEFAULT_LOCK_TIME_IN_SECONDS;

@AttributeDefinition(name = "Lock Update Time",
description = "The interval at which the lock is updated, in seconds")
long lockUpdateTimeInSeconds() default DEFAULT_UPDATE_TIME_IN_SECONDS;

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,6 @@ public final class CASLockFactory implements LockFactory, Closeable
private static final String COLUMN_NODE = "node";
private static final String COLUMN_METADATA = "metadata";
private static final String COLUMN_PRIORITY = "priority";

private static final int LOCK_TIME_IN_SECONDS = 600;
private static final long LOCK_UPDATE_TIME_IN_SECONDS = 60;
private static final int FAILED_LOCK_RETRY_ATTEMPTS =
(int) (LOCK_TIME_IN_SECONDS / LOCK_UPDATE_TIME_IN_SECONDS) - 1;

private static final String TABLE_LOCK = "lock";
private static final String TABLE_LOCK_PRIORITY = "lock_priority";

Expand All @@ -115,13 +109,15 @@ public final class CASLockFactory implements LockFactory, Closeable
private final PreparedStatement myUpdateLockStatement;
private final PreparedStatement myRemoveLockPriorityStatement;
private final LockCache myLockCache;

private final long myLockUpdateTimeInSeconds;
private final int myFailedLockRetryAttempts;
private CASLockFactory(final Builder builder)
masokol marked this conversation as resolved.
Show resolved Hide resolved
{
myStatementDecorator = builder.myStatementDecorator;
myHostStates = builder.myHostStates;
myKeyspaceName = builder.myKeyspaceName;

myLockUpdateTimeInSeconds = builder.myLockUpdateTimeInSeconds;
myFailedLockRetryAttempts = (int) (builder.myLockTimeInSeconds / myLockUpdateTimeInSeconds) - 1;
myExecutor = Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder().setNameFormat("LockRefresher-%d").build());

Expand Down Expand Up @@ -199,7 +195,7 @@ private CASLockFactory(final Builder builder)

myUuid = hostId;

myLockCache = new LockCache(this::doTryLock);
myLockCache = new LockCache(this::doTryLock, builder.myCacheExpiryTimeInSeconds);
}

@Override
Expand Down Expand Up @@ -293,11 +289,17 @@ public static Builder builder()
public static class Builder
{
private static final String DEFAULT_KEYSPACE_NAME = "ecchronos";
private static final long DEFAULT_LOCK_TIME_IN_SECONDS = 600L;
private static final long DEFAULT_LOCK_UPDATE_TIME_IN_SECONDS = 60L;
private static final long DEFAULT_EXPIRY_TIME_IN_SECONDS = 30L;

private NativeConnectionProvider myNativeConnectionProvider;
private HostStates myHostStates;
private StatementDecorator myStatementDecorator;
private String myKeyspaceName = DEFAULT_KEYSPACE_NAME;
private long myLockTimeInSeconds = DEFAULT_LOCK_TIME_IN_SECONDS;
private long myLockUpdateTimeInSeconds = DEFAULT_LOCK_UPDATE_TIME_IN_SECONDS;
private long myCacheExpiryTimeInSeconds = DEFAULT_LOCK_UPDATE_TIME_IN_SECONDS;

public final Builder withNativeConnectionProvider(final NativeConnectionProvider nativeConnectionProvider)
{
Expand All @@ -322,6 +324,21 @@ public final Builder withKeyspaceName(final String keyspaceName)
myKeyspaceName = keyspaceName;
return this;
}
public final Builder withLockTimeInSeconds(final long lockTimeInSeconds)
{
myLockTimeInSeconds = lockTimeInSeconds;
return this;
}
public final Builder withLockUpdateTimeInSeconds (final long lockUpdateTimeInSeconds )
{
myLockUpdateTimeInSeconds = lockUpdateTimeInSeconds;
return this;
}
public final Builder withCacheExpiryInSeconds (final long cacheExpiryInSeconds )
{
myCacheExpiryTimeInSeconds = cacheExpiryInSeconds;
return this;
}

public final CASLockFactory build()
{
Expand Down Expand Up @@ -497,8 +514,8 @@ public boolean lock()
if (tryLock())
{
LOG.trace("Lock for resource {} acquired", myResource);
ScheduledFuture<?> future = myExecutor.scheduleAtFixedRate(this, LOCK_UPDATE_TIME_IN_SECONDS,
LOCK_UPDATE_TIME_IN_SECONDS, TimeUnit.SECONDS);
ScheduledFuture<?> future = myExecutor.scheduleAtFixedRate(this, myLockUpdateTimeInSeconds,
myLockUpdateTimeInSeconds, TimeUnit.SECONDS);
myUpdateFuture.set(future);

return true;
Expand All @@ -520,7 +537,7 @@ public void run()
{
int failedAttempts = myFailedUpdateAttempts.incrementAndGet();

if (failedAttempts >= FAILED_LOCK_RETRY_ATTEMPTS)
if (failedAttempts >= myFailedLockRetryAttempts)
{
LOG.error("Unable to re-lock resource '{}' after {} failed attempts", myResource, failedAttempts);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,12 @@ public final class LockCache
{
private static final Logger LOG = LoggerFactory.getLogger(LockCache.class);

private static final long DEFAULT_EXPIRE_TIME_IN_SECONDS = 30;

private final Cache<LockKey, LockException> myFailureCache;
private final LockSupplier myLockSupplier;

public LockCache(final LockSupplier lockSupplier)
public LockCache(final LockSupplier lockSupplier, final long expireTimeInSeconds)
{
this(lockSupplier, DEFAULT_EXPIRE_TIME_IN_SECONDS, TimeUnit.SECONDS);
this(lockSupplier, expireTimeInSeconds, TimeUnit.SECONDS);
}

LockCache(final LockSupplier lockSupplier, final long expireTime, final TimeUnit expireTimeUnit)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class TestLockCache
@Before
public void setup()
{
myLockCache = new LockCache(mockedLockSupplier);
myLockCache = new LockCache(mockedLockSupplier, 30L);
}

@Test
Expand Down