Fix broken result aggregation when SANDMARK_CUSTOM_NAME is not specified

.github/workflows/main.yml

@@ -49,6 +49,7 @@ jobs:
           ls _results
           cat _results/*
           make json
+          make clean
 
       - name: 5.1.0+stable+parallel
         run: |
@@ -65,6 +66,7 @@ jobs:
           ls _results
           cat _results/*
           make json
+          make clean
 
       - name: 5.1.0+trunk+serial
         run: |
@@ -79,6 +81,7 @@ jobs:
           ls _results
           cat _results/*
           make json
+          make clean
 
       - name: 5.1.0+trunk+parallel
         run: |
@@ -96,6 +99,7 @@ jobs:
           ls _results
           cat _results/*
           make json
+          make clean
 
   build-5-0:
     strategy:

Makefile

@@ -32,9 +32,6 @@ SANDMARK_REMOVE_PACKAGES ?= ""
 # Default list of packages to override
 SANDMARK_OVERRIDE_PACKAGES ?= ""
 
-# Override orun with custom name
-SANDMARK_CUSTOM_NAME ?= ""
-
 # Flag to select whether to use sys_dune_hack
 USE_SYS_DUNE_HACK ?= 0
 
@@ -235,6 +232,7 @@ blah:
 	@echo ${PACKAGES}
 
 ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log_sandmark_hash ocaml-versions/%.json .FORCE
+	$(eval VARIANT_NAME ?= $*)
 	$(eval CONFIG_SWITCH_NAME = $*)
 	$(eval CONFIG_OPTIONS      = $(shell jq -r '.configure // empty' ocaml-versions/$*.json))
 	$(eval CONFIG_RUN_PARAMS   = $(shell jq -r '.runparams // empty' ocaml-versions/$*.json))
@@ -244,7 +242,7 @@ ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log
 	     echo "(context (opam (switch $(CONFIG_SWITCH_NAME)) (name $(CONFIG_SWITCH_NAME)_$$i)))"; \
 	   done } > ocaml-versions/.workspace.$(CONFIG_SWITCH_NAME)
 	opam exec --switch $(CONFIG_SWITCH_NAME) -- rungen _build/$(CONFIG_SWITCH_NAME)_1 $(RUN_CONFIG_JSON) > runs_dune.inc
-	opam exec --switch $(CONFIG_SWITCH_NAME) -- dune build --profile=release --workspace=ocaml-versions/.workspace.$(CONFIG_SWITCH_NAME) @$(BUILD_BENCH_TARGET);
+	opam exec --switch $(CONFIG_SWITCH_NAME) -- dune build --verbose --profile=release --workspace=ocaml-versions/.workspace.$(CONFIG_SWITCH_NAME) @$(BUILD_BENCH_TARGET);
 	@{ if [ "$(BUILD_ONLY)" -eq 0 ]; then												\
 		echo "Executing benchmarks with:";											\
 		echo "  RUN_CONFIG_JSON=${RUN_CONFIG_JSON}";										\
@@ -264,8 +262,8 @@ ocaml-versions/%.bench: depend check-parallel/% filter/% override_packages/% log
 			fi; \
 			done; \
 			header_entry=`jo -p $${s} | jq -c`; \
-			echo "$${header_entry}" > _results/$(SANDMARK_CUSTOM_NAME)_$$i.$(WRAPPER).summary.bench; \
-			find _build/$(CONFIG_SWITCH_NAME)_$$i -name '*.$(WRAPPER).bench' | xargs cat >> _results/$(SANDMARK_CUSTOM_NAME)_$$i.$(WRAPPER).summary.bench;		\
+			echo "$${header_entry}" > _results/$(VARIANT_NAME)_$$i.$(WRAPPER).summary.bench; \
+			find _build/$(CONFIG_SWITCH_NAME)_$$i -name '*.$(WRAPPER).bench' | xargs cat >> _results/$(VARIANT_NAME)_$$i.$(WRAPPER).summary.bench;		\
 		done;															\
 		exit $$ex;														\
 	   else																\

README.md

@@ -210,7 +210,7 @@ benchmarks in byte mode:
 
 ```bash
 $ opam install dune.2.9.0
-$ USE_SYS_DUNE_HACK=1 SANDMARK_CUSTOM_NAME=5.0.0 BUILD_BENCH_TARGET=bytebench \
+$ USE_SYS_DUNE_HACK=1 VARIANT_NAME=5.0.0 BUILD_BENCH_TARGET=bytebench \
     RUN_CONFIG_JSON=run_config_byte.json make ocaml-versions/5.0.0+stable.bench
 ```
 
@@ -387,4 +387,5 @@ work on OS X is to install GNU sed with homebrew and then update the
 | SANDMARK_URL | OCaml compiler source code URL used to build the benchmarks | "" | building compiler and its dependencies |
 | SYS_DUNE_BASE_DIR | Function that returns the path of the system installed dune for use with benchmarking | dune package present in the local opam switch | building compiler and its dependencies |
 | USE_SYS_DUNE_HACK | If the value is 1 then use system installed dune | 0 | building compiler and its dependencies |
+| VARIANT_NAME | Custom name to use for the bench output file | variant from the `ocaml-version/<variant>.json` file name | executing benchmark |
 | WRAPPER | Function to get the wrapper out of `run_<wrapper-name>` | run_orun | executing benchmark |

run_all_custom.sh

@@ -155,7 +155,7 @@ for i in $(seq 0 $((${COUNT} - 1))); do
                 ENVIRONMENT="`echo ${CONFIG_ENVIRONMENT}`" \
                 OCAML_CONFIG_OPTION="`echo ${CONFIG_OPTIONS}`" \
                 OCAML_RUN_PARAM="`echo ${CONFIG_RUN_PARAMS}`" \
-                SANDMARK_CUSTOM_NAME="`echo ${CONFIG_NAME}`" \
+                VARIANT_NAME="`echo ${CONFIG_NAME}`" \
                 SANDMARK_OVERRIDE_PACKAGES="`echo ${CONFIG_OVERRIDE_PACKAGES}`" \
                 SANDMARK_REMOVE_PACKAGES="`echo ${CONFIG_REMOVE_PACKAGES}`" \
                 RUN_BENCH_TARGET="`echo ${CONFIG_RUN_BENCH_TARGET}`" \

run_all_serial.sh

@@ -3,10 +3,8 @@
 TAG='"macro_bench"' make run_config_filtered.json
 
 OPT_WAIT=0 USE_SYS_DUNE_HACK=1 \
-	SANDMARK_CUSTOM_NAME=5.1.0+trunk \
-	RUN_CONFIG_JSON=run_config_filtered.json \
-	make ocaml-versions/5.1.0+trunk.bench
+        RUN_CONFIG_JSON=run_config_filtered.json \
+        make ocaml-versions/5.1.0+trunk.bench
 OPT_WAIT=0 USE_SYS_DUNE_HACK=1 \
-	SANDMARK_CUSTOM_NAME=5.1.0+trunk+decouple_gc \
-	RUN_CONFIG_JSON=run_config_filtered.json \
-	make ocaml-versions/5.1.0+trunk+decouple_gc.bench
+        RUN_CONFIG_JSON=run_config_filtered.json \
+        make ocaml-versions/5.1.0+trunk+decouple_gc.bench