Added BITCOUNT benchmarks

redis · Sep 7, 2024 · 7884357 · 7884357
1 parent b0f898e
commit 7884357
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 41 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.230"
+version = "0.1.234"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py
@@ -370,16 +370,17 @@ def builder_process_stream(
                     deps_list.append("fpconv")
                 redis_temporary_dir = temporary_dir + "/" + redis_dir + "/"
                 logging.info("Using redis temporary dir {}".format(redis_temporary_dir))
-                build_command = "bash -c 'make Makefile.dep && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j'".format(
-                    cpp_compiler,
-                    compiler,
-                    " ".join(deps_list),
-                    build_vars_str,
-                    cpp_compiler,
-                    compiler,
-                    "redis-server",
-                    build_vars_str,
-                )
+                # build_command = "bash -c 'make Makefile.dep && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j'".format(
+                #     cpp_compiler,
+                #     compiler,
+                #     " ".join(deps_list),
+                #     build_vars_str,
+                #     cpp_compiler,
+                #     compiler,
+                #     "redis-server",
+                #     build_vars_str,
+                # )
+                build_command = "sh -c 'make -j'"
                 if b"build_command" in testDetails:
                     build_command = testDetails[b"build_command"].decode()
                 server_name = "redis"

diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py
@@ -1374,7 +1374,7 @@ def process_self_contained_coordinator_stream(
                         (
                             detected_regressions,
                             table_output,
-                            total_improvements,
+                            improvement_list,
                             regressions_list,
                             total_stable,
                             total_unstable,
@@ -1410,37 +1410,44 @@ def process_self_contained_coordinator_stream(
                             running_platform,
                         )
                         total_regressions = len(regressions_list)
+                        total_improvements = len(improvement_list)
                         auto_approve = True
                         grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks"
-
-                        prepare_regression_comment(
-                            auto_approve,
-                            baseline_branch,
-                            baseline_tag,
-                            comparison_branch,
-                            comparison_tag,
-                            contains_regression_comment,
-                            github_pr,
-                            grafana_link_base,
-                            is_actionable_pr,
-                            old_regression_comment_body,
-                            pr_link,
-                            regression_comment,
-                            datasink_conn,
-                            running_platform,
-                            table_output,
-                            tf_github_org,
-                            tf_github_repo,
-                            tf_triggering_env,
-                            total_comparison_points,
-                            total_improvements,
-                            total_regressions,
-                            total_stable,
-                            total_unstable,
-                            verbose,
-                            regressions_percent_lower_limit,
-                            regressions_list,
-                        )
+                        try:
+                            prepare_regression_comment(
+                                auto_approve,
+                                baseline_branch,
+                                baseline_tag,
+                                comparison_branch,
+                                comparison_tag,
+                                contains_regression_comment,
+                                github_pr,
+                                grafana_link_base,
+                                is_actionable_pr,
+                                old_regression_comment_body,
+                                pr_link,
+                                regression_comment,
+                                datasink_conn,
+                                running_platform,
+                                table_output,
+                                tf_github_org,
+                                tf_github_repo,
+                                tf_triggering_env,
+                                total_comparison_points,
+                                total_improvements,
+                                total_regressions,
+                                total_stable,
+                                total_unstable,
+                                verbose,
+                                regressions_percent_lower_limit,
+                                regressions_list,
+                            )
+                        except Exception as e:
+                            logging.error(
+                                "Failed to produce regression comment but continuing... Error: {}".format(
+                                    e.__str__()
+                                )
+                            )
                     logging.info(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"
                     )

diff --git a/...benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml b/...benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml
@@ -0,0 +1,37 @@
+version: 0.4
+name: memtier_benchmark-1key-100M-bits-bitmap-bitcount
+description: Runs memtier_benchmark, for a keyspace length of 1 keys focusing on BITCOUNT performance. The bitmap has a size of 100M bits and approximately 2M bits are set. The size of it is ~15MB
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  init_commands:
+   - '"SETBIT" "users" "100000000" "1"'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --hide-histogram --command "SETBIT users __key__ 1" --key-maximum 100000000 --key-minimum 1 -n 10000 --key-prefix "" --command-key-pattern R --distinct-client-seed --pipeline 10
+  resources:
+    requests:
+      cpus: '2'
+      memory: 1g
+tested-commands:
+- bitcount
+tested-groups:
+- bitmap
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command "BITCOUNT users __key__ -1" --key-maximum 100000000 --key-minimum 1 --key-prefix "" --command-key-pattern R --distinct-client-seed --test-time 120
+  resources:
+    requests:
+      cpus: '2'
+      memory: 2g
+
+priority: 19
diff --git a/...hmarks_specification/test-suites/memtier_benchmark-1key-1Billion-bits-bitmap-bitcount.yml b/...hmarks_specification/test-suites/memtier_benchmark-1key-1Billion-bits-bitmap-bitcount.yml
@@ -0,0 +1,37 @@
+version: 0.4
+name: memtier_benchmark-1key-1Billion-bits-bitmap-bitcount
+description: Runs memtier_benchmark, for a keyspace length of 1 keys focusing on BITCOUNT performance. The bitmap has a size of 1 Billion bits and approximately 20M bits are set. The size of it is ~140MB
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  init_commands:
+   - '"SETBIT" "users" "1000000000" "1"'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --hide-histogram --command "SETBIT users __key__ 1" --key-maximum 1000000000 --key-minimum 1 -n 100000 --key-prefix "" --command-key-pattern R --distinct-client-seed --pipeline 10
+  resources:
+    requests:
+      cpus: '2'
+      memory: 1g
+tested-commands:
+- bitcount
+tested-groups:
+- bitmap
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command "BITCOUNT users __key__ -1" --key-maximum 1000000000 --key-minimum 1 --key-prefix "" --command-key-pattern R --distinct-client-seed --test-time 120
+  resources:
+    requests:
+      cpus: '2'
+      memory: 2g
+
+priority: 19