From 441186e497f7667aa1cbefac8b71562388f769e3 Mon Sep 17 00:00:00 2001 From: Jorgen Ader Date: Mon, 13 May 2019 12:57:22 +0300 Subject: [PATCH] Update Django version to support latest LTS - Add key suffix option to `acquires_lock` --- .travis.yml | 5 ++++- requirements_dev.txt | 1 + setup.py | 3 ++- tg_utils/__init__.py | 2 +- tg_utils/lock.py | 38 ++++++++++++++++++++++++-------------- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 396b100..9679b8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,11 +12,14 @@ env: - DJANGO=1.11 - DJANGO=2.0 - DJANGO=2.1 + - DJANGO=2.2 matrix: exclude: - python: "3.4" env: DJANGO=2.1 + - python: "3.4" + env: DJANGO=2.2 - python: "3.7" env: DJANGO=1.8 - python: "3.7" @@ -45,7 +48,7 @@ deploy: repo: thorgate/tg-utils tags: true python: "3.6" - condition: "$DJANGO = 2.0" + condition: "$DJANGO = 2.2" notifications: email: false diff --git a/requirements_dev.txt b/requirements_dev.txt index 1d5736c..97cb770 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -10,3 +10,4 @@ pytest==2.8.5 pytest-django==2.9.1 django_compressor==2.0 hashids==1.1.0 +python-redis-lock>=3.2.0,<4.0.0 diff --git a/setup.py b/setup.py index 807375e..000ff6c 100755 --- a/setup.py +++ b/setup.py @@ -46,10 +46,11 @@ }, include_package_data=True, install_requires=[ - 'django>=1.8,!=2.1.0,!=2.1.1,<2.2', + 'django>=1.8,!=2.1.0,!=2.1.1,<3.0', ], extras_require={ 'lock': [ + 'redis>=2.10.0', 'python-redis-lock>=3.2.0,<4.0.0', ], 'health_check': [ diff --git a/tg_utils/__init__.py b/tg_utils/__init__.py index 648a4f2..d9b8bea 100644 --- a/tg_utils/__init__.py +++ b/tg_utils/__init__.py @@ -2,4 +2,4 @@ __author__ = 'Thorgate' __email__ = 'code@thorgate.eu' -__version__ = '0.6.1' +__version__ = '0.7.0' diff --git a/tg_utils/lock.py b/tg_utils/lock.py index b1a9887..a1849f3 100644 --- a/tg_utils/lock.py +++ b/tg_utils/lock.py @@ -14,33 +14,37 @@ REDIS_LOCK_URL = getattr(settings, 'REDIS_LOCK_URL', False) -if not REDIS_LOCK_URL: - raise ImproperlyConfigured("To use locking, set REDIS_LOCK_URL in your settings.py") -_redis_conn = StrictRedis.from_url(REDIS_LOCK_URL) -DEFAULT_PREFIX = getattr(settings, 'REDIS_LOCK_DEFAULT_PREFIX', 'acquires_lock_') +DEFAULT_PREFIX = getattr(settings, 'REDIS_LOCK_DEFAULT_PREFIX', 'acquires_lock') logger = logging.getLogger('tg-utils.lock') logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO) +def get_redis_connection(): + if not REDIS_LOCK_URL: + raise ImproperlyConfigured("To use locking, set REDIS_LOCK_URL in your settings.py") + + return StrictRedis.from_url(REDIS_LOCK_URL) + + def get_lock(resource, expires): # Seconds from now on if isinstance(expires, timedelta): expires = expires.total_seconds() - return redis_lock.Lock(_redis_conn, resource, expire=expires) + return redis_lock.Lock(get_redis_connection(), resource, expire=expires) -def acquires_lock(expires, should_fail=True, should_wait=False, resource=None, prefix=DEFAULT_PREFIX): +def acquires_lock(expires, should_fail=True, should_wait=False, resource=None, prefix=DEFAULT_PREFIX, create_id=None): """ Decorator to ensure function only runs when it is unique holder of the resource. Any invocations of the functions before the first is done will raise RuntimeError. - Locks are stored in redis with prefix: `lock:acquires_lock` + Locks are stored in redis with default prefix: `lock:acquires_lock` Arguments: expires(timedelta|int): Expiry time of lock, way more than expected time to run. @@ -49,6 +53,7 @@ def acquires_lock(expires, should_fail=True, should_wait=False, resource=None, p should_wait(bool): Should this task wait for lock to be released. resource(str): Resource identifier, by default taken from function name. prefix(str): Change prefix added to redis key (the 'lock:' part will always be added) + create_id(function): Change suffix added to redis key to lock only specific function call based on arguments. Example: @@ -60,11 +65,6 @@ def acquires_lock(expires, should_fail=True, should_wait=False, resource=None, p def foo(): ... """ - - # Seconds from now on - if isinstance(expires, timedelta): - expires = expires.total_seconds() - # This is just a tiny wrapper around redis_lock # 1) acquire lock or fail # 2) run function @@ -76,12 +76,20 @@ def decorator(f): if resource is None: resource = f.__name__ - resource = '%s%s' % (prefix, resource) + resource = '%s:%s' % (prefix, resource) @wraps(f) def wrapper(*args, **kwargs): + lock_suffix = None + + if create_id: + lock_suffix = create_id(*args, **kwargs) + # The context manager is annoying and always blocking... - lock = redis_lock.Lock(_redis_conn, resource, expire=expires) + lock = get_lock( + resource='%s:%s' % (resource, lock_suffix) if lock_suffix else resource, + expires=expires, + ) lock_acquired = False # Get default lock blocking mode @@ -104,9 +112,11 @@ def wrapper(*args, **kwargs): if not lock.acquire(blocking=is_blocking): if should_fail: raise RuntimeError("Failed to acquire lock: %s" % resource) + logger.warning('Failed to acquire lock: %s', resource) if not should_execute_if_lock_fails: return False + else: lock_acquired = True