From 9cbff116813116db4b6652bb7474dec7257e28ef Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 5 Jan 2024 17:20:12 -0800 Subject: [PATCH 1/5] Add --bench and --warmup flags --- README.md | 10 ++++++++++ run_benchmarks.rb | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/README.md b/README.md index c97ac5f4..979c24d8 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,16 @@ can be controlled by the following environment variables: * `MIN_BENCH_ITRS`: The minimum number of benchmark iterations (default: 10) * `MIN_BENCH_TIME`: The minimum seconds for benchmark (default: 10) +You can also use `--warmup` and `--bench` to set these environment variables: + +```sh +# same as: WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ./run_benchmarks.rb railsbench +./run_benchmarks.rb railsbench --bench=1 + +# same as: WARMUP_ITRS=2 MIN_BENCH_ITRS=3 MIN_BENCH_TIME=0 ./run_benchmarks.rb railsbench +./run_benchmarks.rb railsbench --warmup=2 --bench=3 +``` + There is a handy script for running benchmarks just once using `WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0`, for example with the `--yjit-stats` command-line option: diff --git a/run_benchmarks.rb b/run_benchmarks.rb index 7a8d739b..0b5eac17 100755 --- a/run_benchmarks.rb +++ b/run_benchmarks.rb @@ -347,6 +347,16 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat args.harness = v end + opts.on("--warmup=N", "the number of warmup iterations (default: 15)") do |n| + ENV["WARMUP_ITRS"] = n + end + + opts.on("--bench=N", "the number of benchmark iterations (default: 10)") do |n| + ENV["WARMUP_ITRS"] ||= "0" + ENV["MIN_BENCH_ITRS"] = n + ENV["MIN_BENCH_TIME"] ||= "0" + end + opts.on("--yjit_opts=OPT_STRING", "string of command-line options to run YJIT with (ignored if you use -e)") do |str| args.yjit_opts=str end From 9ddaeae8e0c1a4806dc440eee43c91943d1f73af Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 8 Jan 2024 09:17:22 -0800 Subject: [PATCH 2/5] Drop the --bench=1 shorthand --- README.md | 3 --- run_benchmarks.rb | 1 - 2 files changed, 4 deletions(-) diff --git a/README.md b/README.md index 979c24d8..750902cb 100644 --- a/README.md +++ b/README.md @@ -177,9 +177,6 @@ can be controlled by the following environment variables: You can also use `--warmup` and `--bench` to set these environment variables: ```sh -# same as: WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ./run_benchmarks.rb railsbench -./run_benchmarks.rb railsbench --bench=1 - # same as: WARMUP_ITRS=2 MIN_BENCH_ITRS=3 MIN_BENCH_TIME=0 ./run_benchmarks.rb railsbench ./run_benchmarks.rb railsbench --warmup=2 --bench=3 ``` diff --git a/run_benchmarks.rb b/run_benchmarks.rb index 0b5eac17..dec333fa 100755 --- a/run_benchmarks.rb +++ b/run_benchmarks.rb @@ -352,7 +352,6 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat end opts.on("--bench=N", "the number of benchmark iterations (default: 10)") do |n| - ENV["WARMUP_ITRS"] ||= "0" ENV["MIN_BENCH_ITRS"] = n ENV["MIN_BENCH_TIME"] ||= "0" end From c6a75b207d7a5dc0fbc845f2c8588e8c0be20d80 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 8 Jan 2024 09:52:26 -0800 Subject: [PATCH 3/5] Update the help description of --bench --- run_benchmarks.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_benchmarks.rb b/run_benchmarks.rb index dec333fa..91cc70d9 100755 --- a/run_benchmarks.rb +++ b/run_benchmarks.rb @@ -351,7 +351,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat ENV["WARMUP_ITRS"] = n end - opts.on("--bench=N", "the number of benchmark iterations (default: 10)") do |n| + opts.on("--bench=N", "the number of benchmark iterations for the default harness (default: 10). Also defaults MIN_BENCH_TIME to 0.") do |n| ENV["MIN_BENCH_ITRS"] = n ENV["MIN_BENCH_TIME"] ||= "0" end From 6bde3532ba6ae8518a427d9fd9b1cb9b7d8d500a Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 8 Jan 2024 09:52:35 -0800 Subject: [PATCH 4/5] Update the help description of --warmup Co-authored-by: Benoit Daloze --- run_benchmarks.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_benchmarks.rb b/run_benchmarks.rb index 91cc70d9..23ffcfc6 100755 --- a/run_benchmarks.rb +++ b/run_benchmarks.rb @@ -347,7 +347,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat args.harness = v end - opts.on("--warmup=N", "the number of warmup iterations (default: 15)") do |n| + opts.on("--warmup=N", "the number of warmup iterations for the default harness (default: 15)") do |n| ENV["WARMUP_ITRS"] = n end From eebd59a4b67d56c1b57581f157a5918f37598ac5 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 8 Jan 2024 13:14:13 -0800 Subject: [PATCH 5/5] Add --once option --- README.md | 7 +++++-- run_benchmarks.rb | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 750902cb..e3ecf1b0 100644 --- a/README.md +++ b/README.md @@ -174,14 +174,17 @@ can be controlled by the following environment variables: * `MIN_BENCH_ITRS`: The minimum number of benchmark iterations (default: 10) * `MIN_BENCH_TIME`: The minimum seconds for benchmark (default: 10) -You can also use `--warmup` and `--bench` to set these environment variables: +You can also use `--warmup`, `--bench`, or `--once` to set these environment variables: ```sh # same as: WARMUP_ITRS=2 MIN_BENCH_ITRS=3 MIN_BENCH_TIME=0 ./run_benchmarks.rb railsbench ./run_benchmarks.rb railsbench --warmup=2 --bench=3 + +# same as: WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ./run_benchmarks.rb railsbench +./run_benchmarks.rb railsbench --once ``` -There is a handy script for running benchmarks just once using +There is also a handy script for running benchmarks just once using `WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0`, for example with the `--yjit-stats` command-line option: diff --git a/run_benchmarks.rb b/run_benchmarks.rb index 23ffcfc6..700daf50 100755 --- a/run_benchmarks.rb +++ b/run_benchmarks.rb @@ -356,6 +356,12 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat ENV["MIN_BENCH_TIME"] ||= "0" end + opts.on("--once", "benchmarks only 1 iteration with no warmup for the default harness") do + ENV["WARMUP_ITRS"] = "0" + ENV["MIN_BENCH_ITRS"] = "1" + ENV["MIN_BENCH_TIME"] = "0" + end + opts.on("--yjit_opts=OPT_STRING", "string of command-line options to run YJIT with (ignored if you use -e)") do |str| args.yjit_opts=str end