From e32006c766fe4741c385e7554a9dc4a024b1ac56 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 12 Mar 2025 01:45:12 +0000 Subject: [PATCH 1/3] Extended ZSET benchmarks with ZRANGESTORE and extra ZADD benchmarks --- pyproject.toml | 2 +- ...et-with-5-elements-parsing-float-score.yml | 30 ++++++++++++++++ ...set-with-5-elements-parsing-hexa-score.yml | 30 ++++++++++++++++ ...-600K-elements-zrangestore-1K-elements.yml | 34 +++++++++++++++++++ ...00K-elements-zrangestore-300K-elements.yml | 34 +++++++++++++++++++ 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements.yml diff --git a/pyproject.toml b/pyproject.toml index 8fa7cf0..5585de5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.259" +version = "0.1.263" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score.yml new file mode 100644 index 0000000..e3eb36a --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score +description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different floating-point numbers, gradually increasing in the size of the float to parse. This is a control benchmark for the one named memtier_benchmark-1key-load-zset-with-5-elements-hexa-score +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- zadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 -c 50 -t 4 --command "ZADD zset 2.0 element-1-simple-score 62.0 element-2-medium-score 2047.9999999999998 element-3-long-score 1.0e+30 element-4-very-long-score 1.0e+60 element-5-extra-large-score" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- sorted-set +priority: 12 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score.yml new file mode 100644 index 0000000..30696ee --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score +description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different valid hexadecimal floating-point numbers, gradually increasing in length of the string to parse. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- zadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 -c 50 -t 4 --command "ZADD zset 0x1p+1 element-1-simple-score 0x1.fp+5 element-2-medium-score 0x1.fffffffffffffp+10 element-3-long-score 0x1.ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffp+100 element-4-very-long-score 0x1.fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffp+200 element-5-extra-large-score" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- sorted-set +priority: 12 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements.yml new file mode 100644 index 0000000..8646b25 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements +description: Runs memtier_benchmark, for a keyspace length of 1 key doing zrangestore on a high cardinality sorted set +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '-n allkeys "--data-size" "10" --key-prefix "" "--command" "ZADD zset __key__ __key__" "--command-key-pattern" "P" "-c" "1" "-t" "1" "--hide-histogram" "--key-minimum" "1" "--key-maximum" "600001"' + resources: + requests: + memory: 1g +tested-commands: +- zrangestore +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 -c 1 -t 1 --command "ZRANGESTORE zset1 zset 0 1000" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- sorted-set +priority: 12 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements.yml new file mode 100644 index 0000000..874a15a --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements +description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different floating-point numbers, gradually increasing in the size of the float to parse. This is a control benchmark for the one named memtier_benchmark-1key-load-zset-with-5-elements-hexa-score +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '-n allkeys "--data-size" "10" --key-prefix "" "--command" "ZADD zset __key__ __key__" "--command-key-pattern" "P" "-c" "1" "-t" "1" "--hide-histogram" "--key-minimum" "1" "--key-maximum" "600001"' + resources: + requests: + memory: 1g +tested-commands: +- zrangestore +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 -c 1 -t 1 --command "ZRANGESTORE zset1 zset 0 300000" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- sorted-set +priority: 12 From e9a1be946428b69265e48a9909446684c724bfac Mon Sep 17 00:00:00 2001 From: "Filipe Oliveira (Redis)" Date: Mon, 17 Mar 2025 10:42:35 +0000 Subject: [PATCH 2/3] Fixed failing memtier args test (#281) --- utils/tests/test_runner.py | 16 ++++++++-------- .../test_self_contained_coordinator_memtier.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/tests/test_runner.py b/utils/tests/test_runner.py index 4cb7ab7..f7a21a4 100644 --- a/utils/tests/test_runner.py +++ b/utils/tests/test_runner.py @@ -35,7 +35,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) (_, benchmark_command_str, _) = prepare_memtier_benchmark_parameters( benchmark_config["clientconfig"], @@ -55,7 +55,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time=5' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time=5' ) oss_api_enabled = True (_, benchmark_command_str, _) = prepare_memtier_benchmark_parameters( @@ -69,7 +69,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) oss_api_enabled = False @@ -94,7 +94,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) tls_enabled = True @@ -113,7 +113,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --tls-skip-verify "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --tls --tls-skip-verify "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) tls_skip_verify = False @@ -132,7 +132,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --tls "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) tls_skip_verify = False @@ -153,7 +153,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --tls --cert cert.file --key key.file "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) tls_cacert = "cacert.file" @@ -173,7 +173,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file --cacert cacert.file "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --tls --cert cert.file --key key.file --cacert cacert.file "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index b66b179..f4e5b11 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -1076,7 +1076,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) oss_api_enabled = True ( @@ -1092,7 +1092,7 @@ def test_prepare_memtier_benchmark_parameters(): ) assert ( benchmark_command_str - == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) From 212f690526d3208e17f6f0d925c4b071d577f891 Mon Sep 17 00:00:00 2001 From: "Filipe Oliveira (Redis)" Date: Mon, 17 Mar 2025 15:20:45 +0000 Subject: [PATCH 3/3] Fixed wrong test arg order on redis_benchmarks_specification (#282) * Fixed failing memtier args test * Fixed wrong test arg order on redis_benchmarks_specification/__self_contained_coordinator__/clients.py --- .../__self_contained_coordinator__/clients.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py index e09df38..85d71ca 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py @@ -7,13 +7,13 @@ def prepare_memtier_benchmark_parameters( oss_cluster_api_enabled, ): benchmark_command = [ + "--json-out-file", + local_benchmark_output_filename, full_benchmark_path, "--port", "{}".format(port), "--server", "{}".format(server), - "--json-out-file", - local_benchmark_output_filename, ] if oss_cluster_api_enabled is True: benchmark_command.append("--cluster-mode")