From 2fb63e075d0bcee6cc9d4158748a4456e6acd350 Mon Sep 17 00:00:00 2001 From: Puneeth Chaganti Date: Fri, 15 Jul 2022 11:57:21 +0530 Subject: [PATCH 1/4] Make run_all_*.sh script executable --- run_all_custom.sh | 0 run_all_parallel.sh | 0 run_all_serial.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 run_all_custom.sh mode change 100644 => 100755 run_all_parallel.sh mode change 100644 => 100755 run_all_serial.sh diff --git a/run_all_custom.sh b/run_all_custom.sh old mode 100644 new mode 100755 diff --git a/run_all_parallel.sh b/run_all_parallel.sh old mode 100644 new mode 100755 diff --git a/run_all_serial.sh b/run_all_serial.sh old mode 100644 new mode 100755 From 3644d7c7ae0d94fca864d799010a29f537354528 Mon Sep 17 00:00:00 2001 From: Puneeth Chaganti Date: Mon, 18 Jul 2022 13:08:04 +0530 Subject: [PATCH 2/4] Obtain SANDMARK_CUSTOM_NAME from variant name and make it optional But, also allow overriding it using an environment variable, if required. Also, cleanup on each run in CI to avoid having multiple bench files in the `_results/` directory when we benchmarks are run with different variant names. --- .github/workflows/main.yml | 4 ++++ Makefile | 4 +--- README.md | 1 + run_all_serial.sh | 10 ++++------ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69756d2e73..f29250225d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,6 +49,7 @@ jobs: ls _results cat _results/* make json + make clean - name: 5.1.0+stable+parallel run: | @@ -65,6 +66,7 @@ jobs: ls _results cat _results/* make json + make clean - name: 5.1.0+trunk+serial run: | @@ -79,6 +81,7 @@ jobs: ls _results cat _results/* make json + make clean - name: 5.1.0+trunk+parallel run: | @@ -96,6 +99,7 @@ jobs: ls _results cat _results/* make json + make clean build-5-0: strategy: diff --git a/Makefile b/Makefile index a50d6d64db..b94154ad7c 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,6 @@ SANDMARK_REMOVE_PACKAGES ?= "" # Default list of packages to override SANDMARK_OVERRIDE_PACKAGES ?= "" -# Override orun with custom name -SANDMARK_CUSTOM_NAME ?= "" - # Flag to select whether to use sys_dune_hack USE_SYS_DUNE_HACK ?= 0 @@ -235,6 +232,7 @@ blah: @echo ${PACKAGES} ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log_sandmark_hash ocaml-versions/%.json .FORCE + $(eval SANDMARK_CUSTOM_NAME ?= $*) $(eval CONFIG_SWITCH_NAME = $*) $(eval CONFIG_OPTIONS = $(shell jq -r '.configure // empty' ocaml-versions/$*.json)) $(eval CONFIG_RUN_PARAMS = $(shell jq -r '.runparams // empty' ocaml-versions/$*.json)) diff --git a/README.md b/README.md index 8ca15f89f1..a22ac19db1 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,7 @@ work on OS X is to install GNU sed with homebrew and then update the | PRE_BENCH_EXEC | Any specific commands that needed to be executed before the benchmark. For eg. `PRE_BENCH_EXEC='taskset --cpu-list 3 setarch uname -m --addr-no-randomize'` | null string | executing benchmark | RUN_BENCH_TARGET | The executable to be used to run the benchmarks | `run_orun` | executing benchmark | | RUN_BENCH_TARGET | The executable to be used to run the benchmarks | `run_orun` | executing benchmark | | RUN_CONFIG_JSON | Input file selection that contains the list of benchmarks | `run_config.json` | executing benchmark | +| SANDMARK_CUSTOM_NAME | Custom name to use for the bench output file | variant from the `ocaml-version/.json` file name | executing benchmark | | SANDMARK_DUNE_VERSION | Default dune version to be used | 2.9.0 | building compiler and its dependencies | | SANDMARK_OVERRIDE_PACKAGES | A list of dependency packages with versions that can be overrided (optional) | "" | building compiler and its dependencies | | SANDMARK_REMOVE_PACKAGES | A list of dependency packages to be dynamically removed (optional) | "" | building compiler and its dependencies | diff --git a/run_all_serial.sh b/run_all_serial.sh index def23583bb..bc0b172b99 100755 --- a/run_all_serial.sh +++ b/run_all_serial.sh @@ -3,10 +3,8 @@ TAG='"macro_bench"' make run_config_filtered.json OPT_WAIT=0 USE_SYS_DUNE_HACK=1 \ - SANDMARK_CUSTOM_NAME=5.1.0+trunk \ - RUN_CONFIG_JSON=run_config_filtered.json \ - make ocaml-versions/5.1.0+trunk.bench + RUN_CONFIG_JSON=run_config_filtered.json \ + make ocaml-versions/5.1.0+trunk.bench OPT_WAIT=0 USE_SYS_DUNE_HACK=1 \ - SANDMARK_CUSTOM_NAME=5.1.0+trunk+decouple_gc \ - RUN_CONFIG_JSON=run_config_filtered.json \ - make ocaml-versions/5.1.0+trunk+decouple_gc.bench + RUN_CONFIG_JSON=run_config_filtered.json \ + make ocaml-versions/5.1.0+trunk+decouple_gc.bench From e97dbdfda440b8ac3abb7d0dad1e516a6c62bd99 Mon Sep 17 00:00:00 2001 From: Puneeth Chaganti Date: Tue, 26 Jul 2022 13:25:37 +0530 Subject: [PATCH 3/4] Rename SANDMARK_CUSTOM_NAME to VARIANT_NAME Closes #347 --- Makefile | 6 +++--- README.md | 4 ++-- run_all_custom.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b94154ad7c..7f59a0155f 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,7 @@ blah: @echo ${PACKAGES} ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log_sandmark_hash ocaml-versions/%.json .FORCE - $(eval SANDMARK_CUSTOM_NAME ?= $*) + $(eval VARIANT_NAME ?= $*) $(eval CONFIG_SWITCH_NAME = $*) $(eval CONFIG_OPTIONS = $(shell jq -r '.configure // empty' ocaml-versions/$*.json)) $(eval CONFIG_RUN_PARAMS = $(shell jq -r '.runparams // empty' ocaml-versions/$*.json)) @@ -262,8 +262,8 @@ ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log fi; \ done; \ header_entry=`jo -p $${s} | jq -c`; \ - echo "$${header_entry}" > _results/$(SANDMARK_CUSTOM_NAME)_$$i.$(WRAPPER).summary.bench; \ - find _build/$(CONFIG_SWITCH_NAME)_$$i -name '*.$(WRAPPER).bench' | xargs cat >> _results/$(SANDMARK_CUSTOM_NAME)_$$i.$(WRAPPER).summary.bench; \ + echo "$${header_entry}" > _results/$(VARIANT_NAME)_$$i.$(WRAPPER).summary.bench; \ + find _build/$(CONFIG_SWITCH_NAME)_$$i -name '*.$(WRAPPER).bench' | xargs cat >> _results/$(VARIANT_NAME)_$$i.$(WRAPPER).summary.bench; \ done; \ exit $$ex; \ else \ diff --git a/README.md b/README.md index a22ac19db1..265ed19ccd 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ benchmarks in byte mode: ```bash $ opam install dune.2.9.0 -$ USE_SYS_DUNE_HACK=1 SANDMARK_CUSTOM_NAME=5.0.0 BUILD_BENCH_TARGET=bytebench \ +$ USE_SYS_DUNE_HACK=1 VARIANT_NAME=5.0.0 BUILD_BENCH_TARGET=bytebench \ RUN_CONFIG_JSON=run_config_byte.json make ocaml-versions/5.0.0+stable.bench ``` @@ -381,11 +381,11 @@ work on OS X is to install GNU sed with homebrew and then update the | PRE_BENCH_EXEC | Any specific commands that needed to be executed before the benchmark. For eg. `PRE_BENCH_EXEC='taskset --cpu-list 3 setarch uname -m --addr-no-randomize'` | null string | executing benchmark | RUN_BENCH_TARGET | The executable to be used to run the benchmarks | `run_orun` | executing benchmark | | RUN_BENCH_TARGET | The executable to be used to run the benchmarks | `run_orun` | executing benchmark | | RUN_CONFIG_JSON | Input file selection that contains the list of benchmarks | `run_config.json` | executing benchmark | -| SANDMARK_CUSTOM_NAME | Custom name to use for the bench output file | variant from the `ocaml-version/.json` file name | executing benchmark | | SANDMARK_DUNE_VERSION | Default dune version to be used | 2.9.0 | building compiler and its dependencies | | SANDMARK_OVERRIDE_PACKAGES | A list of dependency packages with versions that can be overrided (optional) | "" | building compiler and its dependencies | | SANDMARK_REMOVE_PACKAGES | A list of dependency packages to be dynamically removed (optional) | "" | building compiler and its dependencies | | SANDMARK_URL | OCaml compiler source code URL used to build the benchmarks | "" | building compiler and its dependencies | | SYS_DUNE_BASE_DIR | Function that returns the path of the system installed dune for use with benchmarking | dune package present in the local opam switch | building compiler and its dependencies | | USE_SYS_DUNE_HACK | If the value is 1 then use system installed dune | 0 | building compiler and its dependencies | +| VARIANT_NAME | Custom name to use for the bench output file | variant from the `ocaml-version/.json` file name | executing benchmark | | WRAPPER | Function to get the wrapper out of `run_` | run_orun | executing benchmark | diff --git a/run_all_custom.sh b/run_all_custom.sh index e7cc095d0e..0b5b7d0fd8 100755 --- a/run_all_custom.sh +++ b/run_all_custom.sh @@ -155,7 +155,7 @@ for i in $(seq 0 $((${COUNT} - 1))); do ENVIRONMENT="`echo ${CONFIG_ENVIRONMENT}`" \ OCAML_CONFIG_OPTION="`echo ${CONFIG_OPTIONS}`" \ OCAML_RUN_PARAM="`echo ${CONFIG_RUN_PARAMS}`" \ - SANDMARK_CUSTOM_NAME="`echo ${CONFIG_NAME}`" \ + VARIANT_NAME="`echo ${CONFIG_NAME}`" \ SANDMARK_OVERRIDE_PACKAGES="`echo ${CONFIG_OVERRIDE_PACKAGES}`" \ SANDMARK_REMOVE_PACKAGES="`echo ${CONFIG_REMOVE_PACKAGES}`" \ RUN_BENCH_TARGET="`echo ${CONFIG_RUN_BENCH_TARGET}`" \ From 6a33f6083c26493a70d7d38a0cb8b00d23b86c64 Mon Sep 17 00:00:00 2001 From: Puneeth Chaganti Date: Mon, 2 Jan 2023 13:32:01 +0530 Subject: [PATCH 4/4] DBG: verbose --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7f59a0155f..9917da8d36 100644 --- a/Makefile +++ b/Makefile @@ -242,7 +242,7 @@ ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log echo "(context (opam (switch $(CONFIG_SWITCH_NAME)) (name $(CONFIG_SWITCH_NAME)_$$i)))"; \ done } > ocaml-versions/.workspace.$(CONFIG_SWITCH_NAME) opam exec --switch $(CONFIG_SWITCH_NAME) -- rungen _build/$(CONFIG_SWITCH_NAME)_1 $(RUN_CONFIG_JSON) > runs_dune.inc - opam exec --switch $(CONFIG_SWITCH_NAME) -- dune build --profile=release --workspace=ocaml-versions/.workspace.$(CONFIG_SWITCH_NAME) @$(BUILD_BENCH_TARGET); + opam exec --switch $(CONFIG_SWITCH_NAME) -- dune build --verbose --profile=release --workspace=ocaml-versions/.workspace.$(CONFIG_SWITCH_NAME) @$(BUILD_BENCH_TARGET); @{ if [ "$(BUILD_ONLY)" -eq 0 ]; then \ echo "Executing benchmarks with:"; \ echo " RUN_CONFIG_JSON=${RUN_CONFIG_JSON}"; \