From 76bf989c19b4087fd169ef2a330704334a5d6193 Mon Sep 17 00:00:00 2001 From: Vladyslav Vildanov <117659936+vladvildanov@users.noreply.github.com> Date: Mon, 2 Sep 2024 11:26:13 +0300 Subject: [PATCH 1/6] Added version restrictions for pytest-asyncio (#3362) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index a8da4b49cd..931784cdaf 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -8,7 +8,7 @@ invoke==2.2.0 mock packaging>=20.4 pytest -pytest-asyncio +pytest-asyncio>=0.23.0,<0.24.0 pytest-cov pytest-profiling pytest-timeout From 147f7cabf0394a04bcb5617ba10e3ea948782ecc Mon Sep 17 00:00:00 2001 From: Dudi <16744955+dudizimber@users.noreply.github.com> Date: Mon, 2 Sep 2024 12:32:54 +0300 Subject: [PATCH 2/6] Add `hostname` field to _parse_node_line (#3343) Co-authored-by: Vladyslav Vildanov <117659936+vladvildanov@users.noreply.github.com> --- redis/_parsers/helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/redis/_parsers/helpers.py b/redis/_parsers/helpers.py index 85b084dfdf..7494c79210 100644 --- a/redis/_parsers/helpers.py +++ b/redis/_parsers/helpers.py @@ -445,9 +445,11 @@ def parse_cluster_info(response, **options): def _parse_node_line(line): line_items = line.split(" ") node_id, addr, flags, master_id, ping, pong, epoch, connected = line.split(" ")[:8] - addr = addr.split("@")[0] + ip = addr.split("@")[0] + hostname = addr.split("@")[1].split(",")[1] if "@" in addr and "," in addr else "" node_dict = { "node_id": node_id, + "hostname": hostname, "flags": flags, "master_id": master_id, "last_ping_sent": ping, @@ -460,7 +462,7 @@ def _parse_node_line(line): if len(line_items) >= 9: slots, migrations = _parse_slots(line_items[8:]) node_dict["slots"], node_dict["migrations"] = slots, migrations - return addr, node_dict + return ip, node_dict def _parse_slots(slot_ranges): From 77e09b7c43d2c89767441c1036e6f27551b06ec9 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Mon, 2 Sep 2024 05:36:47 -0500 Subject: [PATCH 3/6] More docs fixes (#3326) * Docs: Resolve 'Unexpected indentation' warnings * Docs: Resolve 'Unexpected unindent' warnings * Docs: Resolve "more than one target for cross-reference 'Redis'" warnings When Sphinx runs, `TYPE_CHECKING` is not enabled, so the differentiating sync/async `Redis` imports don't happen, and Sphinx appears to be unable to infer which class `"Redis"` should cross-reference. --------- Co-authored-by: Vladyslav Vildanov <117659936+vladvildanov@users.noreply.github.com> --- redis/asyncio/client.py | 10 ++- redis/commands/core.py | 59 ++++++++------ redis/commands/timeseries/commands.py | 110 +++++++++++++------------- 3 files changed, 97 insertions(+), 82 deletions(-) diff --git a/redis/asyncio/client.py b/redis/asyncio/client.py index 1845b7252f..70a5e997ef 100644 --- a/redis/asyncio/client.py +++ b/redis/asyncio/client.py @@ -579,10 +579,12 @@ async def aclose(self, close_connection_pool: Optional[bool] = None) -> None: """ Closes Redis client connection - :param close_connection_pool: decides whether to close the connection pool used - by this Redis client, overriding Redis.auto_close_connection_pool. By default, - let Redis.auto_close_connection_pool decide whether to close the connection - pool. + Args: + close_connection_pool: + decides whether to close the connection pool used by this Redis client, + overriding Redis.auto_close_connection_pool. + By default, let Redis.auto_close_connection_pool decide + whether to close the connection pool. """ conn = self.connection if conn: diff --git a/redis/commands/core.py b/redis/commands/core.py index b356d101ee..91f5a8661b 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -46,8 +46,8 @@ from .helpers import list_or_args if TYPE_CHECKING: - from redis.asyncio.client import Redis as AsyncRedis - from redis.client import Redis + import redis.asyncio.client + import redis.client class ACLCommands(CommandsProtocol): @@ -731,16 +731,19 @@ def client_pause(self, timeout: int, all: bool = True, **kwargs) -> ResponseT: For more information see https://redis.io/commands/client-pause - :param timeout: milliseconds to pause clients - :param all: If true (default) all client commands are blocked. - otherwise, clients are only blocked if they attempt to execute - a write command. + Args: + timeout: milliseconds to pause clients + all: If true (default) all client commands are blocked. + otherwise, clients are only blocked if they attempt to execute + a write command. + For the WRITE mode, some commands have special behavior: - EVAL/EVALSHA: Will block client for all scripts. - PUBLISH: Will block client. - PFCOUNT: Will block client. - WAIT: Acknowledgments will be delayed, so this command will - appear blocked. + + * EVAL/EVALSHA: Will block client for all scripts. + * PUBLISH: Will block client. + * PFCOUNT: Will block client. + * WAIT: Acknowledgments will be delayed, so this command will + appear blocked. """ args = ["CLIENT PAUSE", str(timeout)] if not isinstance(timeout, int): @@ -1439,7 +1442,7 @@ class BitFieldOperation: def __init__( self, - client: Union["Redis", "AsyncRedis"], + client: Union["redis.client.Redis", "redis.asyncio.client.Redis"], key: str, default_overflow: Union[str, None] = None, ): @@ -1583,7 +1586,7 @@ def bitcount( return self.execute_command("BITCOUNT", *params, keys=[key]) def bitfield( - self: Union["Redis", "AsyncRedis"], + self: Union["redis.client.Redis", "redis.asyncio.client.Redis"], key: KeyT, default_overflow: Union[str, None] = None, ) -> BitFieldOperation: @@ -1596,7 +1599,7 @@ def bitfield( return BitFieldOperation(self, key, default_overflow=default_overflow) def bitfield_ro( - self: Union["Redis", "AsyncRedis"], + self: Union["redis.client.Redis", "redis.asyncio.client.Redis"], key: KeyT, encoding: str, offset: BitfieldOffsetT, @@ -5464,7 +5467,7 @@ class Script: An executable Lua script object returned by ``register_script`` """ - def __init__(self, registered_client: "Redis", script: ScriptTextT): + def __init__(self, registered_client: "redis.client.Redis", script: ScriptTextT): self.registered_client = registered_client self.script = script # Precalculate and store the SHA1 hex digest of the script. @@ -5484,7 +5487,7 @@ def __call__( self, keys: Union[Sequence[KeyT], None] = None, args: Union[Iterable[EncodableT], None] = None, - client: Union["Redis", None] = None, + client: Union["redis.client.Redis", None] = None, ): """Execute the script, passing any required ``args``""" keys = keys or [] @@ -5513,7 +5516,11 @@ class AsyncScript: An executable Lua script object returned by ``register_script`` """ - def __init__(self, registered_client: "AsyncRedis", script: ScriptTextT): + def __init__( + self, + registered_client: "redis.asyncio.client.Redis", + script: ScriptTextT, + ): self.registered_client = registered_client self.script = script # Precalculate and store the SHA1 hex digest of the script. @@ -5533,7 +5540,7 @@ async def __call__( self, keys: Union[Sequence[KeyT], None] = None, args: Union[Iterable[EncodableT], None] = None, - client: Union["AsyncRedis", None] = None, + client: Union["redis.asyncio.client.Redis", None] = None, ): """Execute the script, passing any required ``args``""" keys = keys or [] @@ -5758,7 +5765,7 @@ def script_load(self, script: ScriptTextT) -> ResponseT: """ return self.execute_command("SCRIPT LOAD", script) - def register_script(self: "Redis", script: ScriptTextT) -> Script: + def register_script(self: "redis.client.Redis", script: ScriptTextT) -> Script: """ Register a Lua ``script`` specifying the ``keys`` it will touch. Returns a Script object that is callable and hides the complexity of @@ -5772,7 +5779,10 @@ class AsyncScriptCommands(ScriptCommands): async def script_debug(self, *args) -> None: return super().script_debug() - def register_script(self: "AsyncRedis", script: ScriptTextT) -> AsyncScript: + def register_script( + self: "redis.asyncio.client.Redis", + script: ScriptTextT, + ) -> AsyncScript: """ Register a Lua ``script`` specifying the ``keys`` it will touch. Returns a Script object that is callable and hides the complexity of @@ -6415,9 +6425,12 @@ def function_list( ) -> Union[Awaitable[List], List]: """ Return information about the functions and libraries. - :param library: pecify a pattern for matching library names - :param withcode: cause the server to include the libraries source - implementation in the reply + + Args: + + library: specify a pattern for matching library names + withcode: cause the server to include the libraries source implementation + in the reply """ args = ["LIBRARYNAME", library] if withcode: diff --git a/redis/commands/timeseries/commands.py b/redis/commands/timeseries/commands.py index f8dfe8b5c0..b0cb864237 100644 --- a/redis/commands/timeseries/commands.py +++ b/redis/commands/timeseries/commands.py @@ -60,17 +60,17 @@ def create( duplicate_policy: Policy for handling multiple samples with identical timestamps. Can be one of: - - 'block': An error will occur and the new value will be ignored. - - 'first': Ignore the new value. - - 'last': Override with the latest value. - - 'min': Only override if the value is lower than the existing - value. - - 'max': Only override if the value is higher than the existing - value. - - 'sum': If a previous sample exists, add the new sample to it so - that the updated value is equal to (previous + new). If no - previous sample exists, set the updated value equal to the new - value. + + - 'block': An error will occur and the new value will be ignored. + - 'first': Ignore the new value. + - 'last': Override with the latest value. + - 'min': Only override if the value is lower than the existing value. + - 'max': Only override if the value is higher than the existing value. + - 'sum': If a previous sample exists, add the new sample to it so + that the updated value is equal to (previous + new). If no + previous sample exists, set the updated value equal to the new + value. + ignore_max_time_diff: A non-negative integer value, in milliseconds, that sets an ignore threshold for added timestamps. If the difference between the last @@ -130,17 +130,17 @@ def alter( duplicate_policy: Policy for handling multiple samples with identical timestamps. Can be one of: - - 'block': An error will occur and the new value will be ignored. - - 'first': Ignore the new value. - - 'last': Override with the latest value. - - 'min': Only override if the value is lower than the existing - value. - - 'max': Only override if the value is higher than the existing - value. - - 'sum': If a previous sample exists, add the new sample to it so - that the updated value is equal to (previous + new). If no - previous sample exists, set the updated value equal to the new - value. + + - 'block': An error will occur and the new value will be ignored. + - 'first': Ignore the new value. + - 'last': Override with the latest value. + - 'min': Only override if the value is lower than the existing value. + - 'max': Only override if the value is higher than the existing value. + - 'sum': If a previous sample exists, add the new sample to it so + that the updated value is equal to (previous + new). If no + previous sample exists, set the updated value equal to the new + value. + ignore_max_time_diff: A non-negative integer value, in milliseconds, that sets an ignore threshold for added timestamps. If the difference between the last @@ -210,17 +210,17 @@ def add( duplicate_policy: Policy for handling multiple samples with identical timestamps. Can be one of: - - 'block': An error will occur and the new value will be ignored. - - 'first': Ignore the new value. - - 'last': Override with the latest value. - - 'min': Only override if the value is lower than the existing - value. - - 'max': Only override if the value is higher than the existing - value. - - 'sum': If a previous sample exists, add the new sample to it so - that the updated value is equal to (previous + new). If no - previous sample exists, set the updated value equal to the new - value. + + - 'block': An error will occur and the new value will be ignored. + - 'first': Ignore the new value. + - 'last': Override with the latest value. + - 'min': Only override if the value is lower than the existing value. + - 'max': Only override if the value is higher than the existing value. + - 'sum': If a previous sample exists, add the new sample to it so + that the updated value is equal to (previous + new). If no + previous sample exists, set the updated value equal to the new + value. + ignore_max_time_diff: A non-negative integer value, in milliseconds, that sets an ignore threshold for added timestamps. If the difference between the last @@ -331,17 +331,17 @@ def incrby( duplicate_policy: Policy for handling multiple samples with identical timestamps. Can be one of: - - 'block': An error will occur and the new value will be ignored. - - 'first': Ignore the new value. - - 'last': Override with the latest value. - - 'min': Only override if the value is lower than the existing - value. - - 'max': Only override if the value is higher than the existing - value. - - 'sum': If a previous sample exists, add the new sample to it so - that the updated value is equal to (previous + new). If no - previous sample exists, set the updated value equal to the new - value. + + - 'block': An error will occur and the new value will be ignored. + - 'first': Ignore the new value. + - 'last': Override with the latest value. + - 'min': Only override if the value is lower than the existing value. + - 'max': Only override if the value is higher than the existing value. + - 'sum': If a previous sample exists, add the new sample to it so + that the updated value is equal to (previous + new). If no + previous sample exists, set the updated value equal to the new + value. + ignore_max_time_diff: A non-negative integer value, in milliseconds, that sets an ignore threshold for added timestamps. If the difference between the last @@ -423,17 +423,17 @@ def decrby( duplicate_policy: Policy for handling multiple samples with identical timestamps. Can be one of: - - 'block': An error will occur and the new value will be ignored. - - 'first': Ignore the new value. - - 'last': Override with the latest value. - - 'min': Only override if the value is lower than the existing - value. - - 'max': Only override if the value is higher than the existing - value. - - 'sum': If a previous sample exists, add the new sample to it so - that the updated value is equal to (previous + new). If no - previous sample exists, set the updated value equal to the new - value. + + - 'block': An error will occur and the new value will be ignored. + - 'first': Ignore the new value. + - 'last': Override with the latest value. + - 'min': Only override if the value is lower than the existing value. + - 'max': Only override if the value is higher than the existing value. + - 'sum': If a previous sample exists, add the new sample to it so + that the updated value is equal to (previous + new). If no + previous sample exists, set the updated value equal to the new + value. + ignore_max_time_diff: A non-negative integer value, in milliseconds, that sets an ignore threshold for added timestamps. If the difference between the last From c6faa6597e7f1e64bd8632dd999ed0afc3dd97bc Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Mon, 2 Sep 2024 06:31:04 -0500 Subject: [PATCH 4/6] Ignore a coverage warning re: sysmon on Pythons < 3.12 (#3334) Co-authored-by: Vladyslav Vildanov <117659936+vladvildanov@users.noreply.github.com> --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index 9db630e5b1..990968d6f9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,3 +14,5 @@ timeout = 30 filterwarnings = always ignore:RedisGraph support is deprecated as of Redis Stack 7.2:DeprecationWarning + # Ignore a coverage warning when COVERAGE_CORE=sysmon for Pythons < 3.12. + ignore:sys.monitoring isn't available:coverage.exceptions.CoverageWarning From 93b9d8536190842185729b9bac68c7f89e4f32c5 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Mon, 2 Sep 2024 06:42:59 -0500 Subject: [PATCH 5/6] Delete the first-defined (and thus "duplicate") `Script` class (#3333) The second definition was copied over the first definition, with the following changes: * The type annotations were copied to the second definition * The mutable default arguments to the `keys` and `args` parameters were replaced with `None`, as is best-practice. Closes #3332 Co-authored-by: Vladyslav Vildanov <117659936+vladvildanov@users.noreply.github.com> --- CHANGES | 1 + redis/commands/core.py | 80 ++++++++++-------------------------------- 2 files changed, 20 insertions(+), 61 deletions(-) diff --git a/CHANGES b/CHANGES index f0d75a45ce..8750128b05 100644 --- a/CHANGES +++ b/CHANGES @@ -66,6 +66,7 @@ * Prevent async ClusterPipeline instances from becoming "false-y" in case of empty command stack (#3061) * Close Unix sockets if the connection attempt fails. This prevents `ResourceWarning`s. (#3314) * Close SSL sockets if the connection attempt fails, or if validations fail. (#3317) + * Eliminate mutable default arguments in the `redis.commands.core.Script` class. (#3332) * 4.1.3 (Feb 8, 2022) * Fix flushdb and flushall (#1926) diff --git a/redis/commands/core.py b/redis/commands/core.py index 91f5a8661b..d46e55446c 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -5475,11 +5475,7 @@ def __init__(self, registered_client: "redis.client.Redis", script: ScriptTextT) if isinstance(script, str): # We need the encoding from the client in order to generate an # accurate byte representation of the script - try: - encoder = registered_client.connection_pool.get_encoder() - except AttributeError: - # Cluster - encoder = registered_client.get_encoder() + encoder = self.get_encoder() script = encoder.encode(script) self.sha = hashlib.sha1(script).hexdigest() @@ -5510,6 +5506,24 @@ def __call__( self.sha = client.script_load(self.script) return client.evalsha(self.sha, len(keys), *args) + def get_encoder(self): + """Get the encoder to encode string scripts into bytes.""" + try: + return self.registered_client.get_encoder() + except AttributeError: + # DEPRECATED + # In version <=4.1.2, this was the code we used to get the encoder. + # However, after 4.1.2 we added support for scripting in clustered + # redis. ClusteredRedis doesn't have a `.connection_pool` attribute + # so we changed the Script class to use + # `self.registered_client.get_encoder` (see above). + # However, that is technically a breaking change, as consumers who + # use Scripts directly might inject a `registered_client` that + # doesn't have a `.get_encoder` field. This try/except prevents us + # from breaking backward-compatibility. Ideally, it would be + # removed in the next major release. + return self.registered_client.connection_pool.get_encoder() + class AsyncScript: """ @@ -6293,62 +6307,6 @@ def command(self) -> ResponseT: return self.execute_command("COMMAND") -class Script: - """ - An executable Lua script object returned by ``register_script`` - """ - - def __init__(self, registered_client, script): - self.registered_client = registered_client - self.script = script - # Precalculate and store the SHA1 hex digest of the script. - - if isinstance(script, str): - # We need the encoding from the client in order to generate an - # accurate byte representation of the script - encoder = self.get_encoder() - script = encoder.encode(script) - self.sha = hashlib.sha1(script).hexdigest() - - def __call__(self, keys=[], args=[], client=None): - "Execute the script, passing any required ``args``" - if client is None: - client = self.registered_client - args = tuple(keys) + tuple(args) - # make sure the Redis server knows about the script - from redis.client import Pipeline - - if isinstance(client, Pipeline): - # Make sure the pipeline can register the script before executing. - client.scripts.add(self) - try: - return client.evalsha(self.sha, len(keys), *args) - except NoScriptError: - # Maybe the client is pointed to a different server than the client - # that created this instance? - # Overwrite the sha just in case there was a discrepancy. - self.sha = client.script_load(self.script) - return client.evalsha(self.sha, len(keys), *args) - - def get_encoder(self): - """Get the encoder to encode string scripts into bytes.""" - try: - return self.registered_client.get_encoder() - except AttributeError: - # DEPRECATED - # In version <=4.1.2, this was the code we used to get the encoder. - # However, after 4.1.2 we added support for scripting in clustered - # redis. ClusteredRedis doesn't have a `.connection_pool` attribute - # so we changed the Script class to use - # `self.registered_client.get_encoder` (see above). - # However, that is technically a breaking change, as consumers who - # use Scripts directly might inject a `registered_client` that - # doesn't have a `.get_encoder` field. This try/except prevents us - # from breaking backward-compatibility. Ideally, it would be - # removed in the next major release. - return self.registered_client.connection_pool.get_encoder() - - class AsyncModuleCommands(ModuleCommands): async def command_info(self) -> None: return super().command_info() From b4fcbaa1ea20bbf3997b81221556e06f7aae2017 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Mon, 2 Sep 2024 07:35:36 -0500 Subject: [PATCH 6/6] Catch a known `DeprecationWarning` when calling `.close()` (#3335) Co-authored-by: Vladyslav Vildanov <117659936+vladvildanov@users.noreply.github.com> --- tests/test_asyncio/test_cluster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index c16272bb5b..fefa4ef8f9 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -313,7 +313,8 @@ async def mock_aclose(): called += 1 with mock.patch.object(cluster, "aclose", mock_aclose): - await cluster.close() + with pytest.warns(DeprecationWarning, match=r"Use aclose\(\) instead"): + await cluster.close() assert called == 1 await cluster.aclose()