From 7884357b7e225c0bc7e2bf4c960f22df5e4bc7c2 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Sat, 7 Sep 2024 12:03:17 +0100 Subject: [PATCH] Added BITCOUNT benchmarks --- pyproject.toml | 2 +- .../__builder__/builder.py | 21 +++--- .../self_contained_coordinator.py | 67 ++++++++++--------- ...nchmark-1key-100M-bits-bitmap-bitcount.yml | 37 ++++++++++ ...ark-1key-1Billion-bits-bitmap-bitcount.yml | 37 ++++++++++ 5 files changed, 123 insertions(+), 41 deletions(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-1Billion-bits-bitmap-bitcount.yml diff --git a/pyproject.toml b/pyproject.toml index 91e1bb8..236d0c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.230" +version = "0.1.234" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 4cfcec1..616ee57 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -370,16 +370,17 @@ def builder_process_stream( deps_list.append("fpconv") redis_temporary_dir = temporary_dir + "/" + redis_dir + "/" logging.info("Using redis temporary dir {}".format(redis_temporary_dir)) - build_command = "bash -c 'make Makefile.dep && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j'".format( - cpp_compiler, - compiler, - " ".join(deps_list), - build_vars_str, - cpp_compiler, - compiler, - "redis-server", - build_vars_str, - ) + # build_command = "bash -c 'make Makefile.dep && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j'".format( + # cpp_compiler, + # compiler, + # " ".join(deps_list), + # build_vars_str, + # cpp_compiler, + # compiler, + # "redis-server", + # build_vars_str, + # ) + build_command = "sh -c 'make -j'" if b"build_command" in testDetails: build_command = testDetails[b"build_command"].decode() server_name = "redis" diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index f5f3366..2387f98 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -1374,7 +1374,7 @@ def process_self_contained_coordinator_stream( ( detected_regressions, table_output, - total_improvements, + improvement_list, regressions_list, total_stable, total_unstable, @@ -1410,37 +1410,44 @@ def process_self_contained_coordinator_stream( running_platform, ) total_regressions = len(regressions_list) + total_improvements = len(improvement_list) auto_approve = True grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks" - - prepare_regression_comment( - auto_approve, - baseline_branch, - baseline_tag, - comparison_branch, - comparison_tag, - contains_regression_comment, - github_pr, - grafana_link_base, - is_actionable_pr, - old_regression_comment_body, - pr_link, - regression_comment, - datasink_conn, - running_platform, - table_output, - tf_github_org, - tf_github_repo, - tf_triggering_env, - total_comparison_points, - total_improvements, - total_regressions, - total_stable, - total_unstable, - verbose, - regressions_percent_lower_limit, - regressions_list, - ) + try: + prepare_regression_comment( + auto_approve, + baseline_branch, + baseline_tag, + comparison_branch, + comparison_tag, + contains_regression_comment, + github_pr, + grafana_link_base, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + datasink_conn, + running_platform, + table_output, + tf_github_org, + tf_github_repo, + tf_triggering_env, + total_comparison_points, + total_improvements, + total_regressions, + total_stable, + total_unstable, + verbose, + regressions_percent_lower_limit, + regressions_list, + ) + except Exception as e: + logging.error( + "Failed to produce regression comment but continuing... Error: {}".format( + e.__str__() + ) + ) logging.info( f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}" ) diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml new file mode 100644 index 0000000..899c834 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml @@ -0,0 +1,37 @@ +version: 0.4 +name: memtier_benchmark-1key-100M-bits-bitmap-bitcount +description: Runs memtier_benchmark, for a keyspace length of 1 keys focusing on BITCOUNT performance. The bitmap has a size of 100M bits and approximately 2M bits are set. The size of it is ~15MB +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + init_commands: + - '"SETBIT" "users" "100000000" "1"' + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --hide-histogram --command "SETBIT users __key__ 1" --key-maximum 100000000 --key-minimum 1 -n 10000 --key-prefix "" --command-key-pattern R --distinct-client-seed --pipeline 10 + resources: + requests: + cpus: '2' + memory: 1g +tested-commands: +- bitcount +tested-groups: +- bitmap +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command "BITCOUNT users __key__ -1" --key-maximum 100000000 --key-minimum 1 --key-prefix "" --command-key-pattern R --distinct-client-seed --test-time 120 + resources: + requests: + cpus: '2' + memory: 2g + +priority: 19 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-1Billion-bits-bitmap-bitcount.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-1Billion-bits-bitmap-bitcount.yml new file mode 100644 index 0000000..fffb070 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-1Billion-bits-bitmap-bitcount.yml @@ -0,0 +1,37 @@ +version: 0.4 +name: memtier_benchmark-1key-1Billion-bits-bitmap-bitcount +description: Runs memtier_benchmark, for a keyspace length of 1 keys focusing on BITCOUNT performance. The bitmap has a size of 1 Billion bits and approximately 20M bits are set. The size of it is ~140MB +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + init_commands: + - '"SETBIT" "users" "1000000000" "1"' + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --hide-histogram --command "SETBIT users __key__ 1" --key-maximum 1000000000 --key-minimum 1 -n 100000 --key-prefix "" --command-key-pattern R --distinct-client-seed --pipeline 10 + resources: + requests: + cpus: '2' + memory: 1g +tested-commands: +- bitcount +tested-groups: +- bitmap +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command "BITCOUNT users __key__ -1" --key-maximum 1000000000 --key-minimum 1 --key-prefix "" --command-key-pattern R --distinct-client-seed --test-time 120 + resources: + requests: + cpus: '2' + memory: 2g + +priority: 19