Improve benchmark quality

Makefile

@@ -45,39 +45,18 @@ checkdoc:
 	  --eval "(setq byte-compile-error-on-warn t)" \
 	  -f batch-byte-compile $<
 
-bench = time $(EMACS) -Q -nw $(3) \
---eval '(setq enable-local-variables nil)' \
---eval '(setq large-file-warning-threshold nil)' \
---eval '(switch-to-buffer (find-file-literally "$(1)"))' \
---eval $(2) \
---eval '(condition-case err \
-(with-current-buffer (current-buffer) \
-(setq font-lock-major-mode nil) \
-(syntax-ppss-flush-cache -1) \
-(font-lock-set-defaults) \
-(save-excursion \
-(font-lock-fontify-region (point-min) (point-max)))) \
-((debug error) (kill-emacs (error-message-string err))))' \
---eval '(goto-char (point-max))' \
---eval '(kill-emacs)'
-
 LARGE_JSON_FILE := test-assets/large-json-file.json
 ${LARGE_JSON_FILE}:
 	curl 'https://raw.githubusercontent.com/pulumi/pulumi-azure-native/master/provider/cmd/pulumi-resource-azure-native/schema.json' > ${LARGE_JSON_FILE}
 
-bench-base: ${LARGE_JSON_FILE} jsonian.elc
-
-bench-jsonian: bench-base
-	$(call bench,${LARGE_JSON_FILE}, "(progn (require 'jsonian) (jsonian-mode))", -L .)
-
-bench-json-mode: bench-base
-	$(call bench,${LARGE_JSON_FILE}, "(progn (require 'json-mode) (json-mode))", -L ../json-mode -L ../json-snatcher -L ../json-reformat)
-
-bench-javascript: bench-base
-	$(call bench,${LARGE_JSON_FILE}, "(javascript-mode)",)
+README.md: bench/markdown.md
+	cp $@ $@.backup
+	rg -U '(?s)<!--BENCHMARK_START-->.*<!--BENCHMARK_END-->' -r "$(cat bench/markdown.md)" --passthru < $@ > $@.new
+	mv $@.new $@
 
-bench-fundamental: bench-base
-	$(call bench,${LARGE_JSON_FILE},"(fundamental-mode)",)
+bench/markdown.md: bench/font-lock.md
+	EXPORT="$@" ./bench/markdown.sh
 
-bench-prog: bench-base
-	$(call bench,${LARGE_JSON_FILE},"(prog-mode)",)
+bench/font-lock.md: ${LARGE_JSON_FILE} jsonian.elc
+	hyperfine --version
+	EMACS="'${EMACS}'" FILE="${LARGE_JSON_FILE}" EXPORT="$@" ./bench/font-lock.sh

README.md

@@ -178,7 +178,8 @@ buffer. When the element is selected, jump to that point in the buffer.
 
 Enable `jsonian-mode` for all checkers where `json-mode` is enabled.
 
-## Speed comparison against other modes
+<!--BENCHMARK_START-->
+## Benchmarks
 
 Part of the promise of `jsonian` is that it will be performant on large files. A
 primitive benchmark is included in the `Makefile`. It opens a very very large
@@ -212,6 +213,7 @@ Notes:
 1. Both `jsonian` and `json-mode` were byte-compiled for this benchmark. Byte
    compiling `jsonian` shaves 6 seconds off of this benchmark.
 2. These benchmarks were taken on a 2.6 GHz 6-Core Intel i7 running macOS Monterey.
+<!--BENCHMARK_END-->
 
 ## Contributing
 

bench/font-lock.md

@@ -0,0 +1,7 @@
+| Command | Mean [s] | Min [s] | Max [s] | Relative |
+|:---|---:|---:|---:|---:|
+| `jsonian-mode` | 2.222 ± 0.007 | 2.213 | 2.239 | 1.68 ± 0.01 |
+| `json-mode` | 3.689 ± 0.102 | 3.622 | 3.910 | 2.78 ± 0.08 |
+| `javascript-mode` | 13.138 ± 0.062 | 13.042 | 13.224 | 9.91 ± 0.07 |
+| `fundamental-mode` | 1.325 ± 0.007 | 1.320 | 1.341 | 1.00 |
+| `prog-mode` | 1.396 ± 0.005 | 1.392 | 1.406 | 1.05 ± 0.01 |

bench/font-lock.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env sh
+
+# EMACS should be set to the binary to invoke.
+# FILE should be set to the file to test against.
+
+bench() {
+    echo "$EMACS -Q --batch -nw  \
+-L . -L ../json-mode -L ../json-snatcher -L ../json-reformat \
+--eval '(setq enable-local-variables nil)' \
+--eval '(setq large-file-warning-threshold nil)' \
+--eval '(switch-to-buffer (find-file-literally \"$FILE\"))' \
+--eval \"$1\" \
+--eval '(condition-case err \
+(with-current-buffer (current-buffer) \
+(setq font-lock-major-mode nil) \
+(syntax-ppss-flush-cache -1) \
+(font-lock-set-defaults) \
+(save-excursion \
+(font-lock-fontify-region (point-min) (point-max)))) \
+((debug error) (kill-emacs (error-message-string err))))' \
+--eval '(goto-char (point-max))' \
+--eval '(kill-emacs)'"
+}
+
+hyperfine --export-markdown "$EXPORT" --show-output \
+          --command-name "fundamental-mode" "$(bench "(fundamental-mode)")" \
+          --command-name "prog-mode" "$(bench "(prog-mode)")" \
+          --command-name "jsonian-mode" "$(bench "(progn (require 'jsonian) (jsonian-mode))")" \
+          --command-name "json-mode" "$(bench "(progn (require 'json-mode) (json-mode))")" \
+          --command-name "javascript-mode" "$(bench "(javascript-mode)")"

bench/markdown.md

@@ -0,0 +1,19 @@
+
+## Benchmarks
+
+The original reason I wrote jsonian is that I needed to read and naviage very large JSON
+files, and Emacs was slowing me down. To keep jsonian fast, I maintain benchmarks of
+jsonian doing real world tasks.
+
+### `font-lock`ing a large buffer
+
+This benchmark opens a very large (42M) JSON file, then forces Emacs to fontify it. It
+finally moves point to the end of the file and exits.
+
+| Command | Mean [s] | Min [s] | Max [s] | Relative |
+|:---|---:|---:|---:|---:|
+| `jsonian-mode` | 2.222 ± 0.007 | 2.213 | 2.239 | 1.68 ± 0.01 |
+| `json-mode` | 3.689 ± 0.102 | 3.622 | 3.910 | 2.78 ± 0.08 |
+| `javascript-mode` | 13.138 ± 0.062 | 13.042 | 13.224 | 9.91 ± 0.07 |
+| `fundamental-mode` | 1.325 ± 0.007 | 1.320 | 1.341 | 1.00 |
+| `prog-mode` | 1.396 ± 0.005 | 1.392 | 1.406 | 1.05 ± 0.01 |

bench/markdown.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env sh
+
+MESSAGE=$(cat <<-EOF
+
+## Benchmarks
+
+The original reason I wrote jsonian is that I needed to read and naviage very large JSON
+files, and Emacs was slowing me down. To keep jsonian fast, I maintain benchmarks of
+jsonian doing real world tasks.
+
+### \`font-lock\`ing a large buffer
+
+This benchmark opens a very large (42M) JSON file, then forces Emacs to fontify it. It
+finally moves point to the end of the file and exits.
+
+$(sed 's/Command/Package/g' < bench/font-lock.md)
+
+We can use this benchmark to derive how long different parts of the proces take.
+
+- Fundamental mode is the lower limit. This is the time Emacs spends processing the
+  buffer, parsing sexps, etc.
+
+- We see that \`prog-mode\` doesn't do much more then \`fundamental-mode\`, which makes
+  sense.
+
+- Applying JSON formatting take at most \`jsonian-mode\` - \`prog-mode\`.
+
+Notes:
+
+- Both \`jsonian\` and \`json-mode\` were byte-compiled for the benchmark.
+
+EOF
+       )
+
+echo "$MESSAGE" > "$EXPORT"