Output SVG files

Author: mdboom

DEVELOPER.md

@@ -104,7 +104,7 @@ With this base, there are files with the following suffixes:
 
 - `.json`: The raw results from `pyperformance`.
 - `-vs-{base}.md`: A table comparing the results against the given base, as returned by `pyperf compare_to`.
-- `-vs-{base}.png`: A set of violin plots with the distribution of differences for each benchmark.
+- `-vs-{base}.svg`: A set of violin plots with the distribution of differences for each benchmark.
 - `-pystats.json`: The raw results from a pystats run.
 - `-pystats.md`: The results of a pystats run, summarized in human-readable form.
 

bench_runner/plot.py

@@ -7,13 +7,15 @@
 from operator import attrgetter
 from pathlib import Path
 import re
+import tempfile
 from typing import Callable, Iterable
 
 
 from matplotlib import pyplot as plt
 import matplotlib
 import numpy as np
 import rich_argparse
+from scour import scour
 import simdjson
 
 
@@ -46,6 +48,26 @@
 }
 
 
+def savefig(output_filename: Path, **kwargs):
+    class Options:
+        quiet = True
+        remove_descriptive_elements = True
+        enable_comment_stripping = True
+        indent_type = "none"
+        strip_ids = True
+        shorten_ids = True
+        digits = 3
+
+    plt.savefig(output_filename, **kwargs)
+    if output_filename.suffix == ".svg":
+        with tempfile.NamedTemporaryFile(delete=False) as tmp:
+            with open(output_filename) as fd:
+                scour.start(Options(), fd, tmp)
+            output_filename.unlink()
+            Path(tmp.name).rename(output_filename)
+    plt.close()
+
+
 @functools.cache
 def get_plot_config():
     with Path("plotconfig.json").open() as fd:
@@ -153,8 +175,7 @@ def plot_diff(
     axs.grid()
     axs.set_title(title)
 
-    plt.savefig(output_filename)
-    plt.close()
+    savefig(output_filename)
 
 
 def get_micro_version(version: str) -> str:
@@ -323,8 +344,7 @@ def get_comparison_value(ref, r, base):
 
     fig.suptitle(title)
 
-    plt.savefig(output_filename, dpi=150)
-    plt.close()
+    savefig(output_filename, dpi=150)
 
     with data_cache.open("w") as fd:
         simdjson.dump(data, fd, indent=2)
@@ -419,8 +439,7 @@ def get_comparison_value(ref, r):
     for ax in axs:
         ax.set_xlim((minx, maxx))
 
-    plt.savefig(output_filename, dpi=150)
-    plt.close()
+    savefig(output_filename, dpi=150)
 
     with data_cache.open("w") as fd:
         simdjson.dump(data, fd, indent=2)

bench_runner/result.py

@@ -104,9 +104,9 @@ def get_files(self) -> Iterable[tuple[Callable, str, str]]:
         if self.base_filename is None:
             return
         yield (self.write_table, ".md", "table")
-        yield (self.write_timing_plot, ".png", "time plot")
+        yield (self.write_timing_plot, ".svg", "time plot")
         if self.head.system != "windows" and self.base == "base":
-            yield (self.write_memory_plot, "-mem.png", "memory plot")
+            yield (self.write_memory_plot, "-mem.svg", "memory plot")
 
     @functools.cached_property
     def _contents(self) -> str | None:
@@ -534,9 +534,9 @@ def result_info(self) -> tuple[str | None, str | None]:
                 return ("pystats raw", None)
             case (["vs", base], ".md"):
                 return ("table", base)
-            case (["vs", base], ".png"):
+            case (["vs", base], ".svg"):
                 return ("time plot", base)
-            case (["vs", base, "mem"], ".png"):
+            case (["vs", base, "mem"], ".svg"):
                 return ("memory plot", base)
         raise ValueError(
             f"Unknown result type (extra={self.extra} suffix={self.suffix})"

bench_runner/scripts/generate_results.py

@@ -401,17 +401,17 @@ def _main(repo_dir: Path, force: bool = False, bases: Sequence[str] | None = Non
         [
             (
                 plot.longitudinal_plot,
-                (benchmarking_results, repo_dir / "longitudinal.png"),
+                (benchmarking_results, repo_dir / "longitudinal.svg"),
                 {},
             ),
             (
                 plot.flag_effect_plot,
-                (benchmarking_results, repo_dir / "configs.png"),
+                (benchmarking_results, repo_dir / "configs.svg"),
                 {},
             ),
             (
                 plot.longitudinal_plot,
-                (memory_benchmarking_results, repo_dir / "memory_long.png"),
+                (memory_benchmarking_results, repo_dir / "memory_long.svg"),
                 dict(
                     getter=lambda r: r.memory_change_float,
                     differences=("less", "more"),
@@ -420,7 +420,7 @@ def _main(repo_dir: Path, force: bool = False, bases: Sequence[str] | None = Non
             ),
             (
                 plot.flag_effect_plot,
-                (memory_benchmarking_results, repo_dir / "memory_configs.png"),
+                (memory_benchmarking_results, repo_dir / "memory_configs.svg"),
                 dict(
                     getter=lambda r: r.memory_change_float,
                     differences=("less", "more"),

bench_runner/scripts/profiling_plot.py

@@ -350,7 +350,7 @@ def _main(input_dir: Path, output_prefix: Path):
     ax.legend(bbox_to_anchor=(1.05, 1.0), loc="upper left")
     ax.set_xlim([0, 1])
 
-    fig.savefig(output_prefix.with_suffix(".png"))
+    fig.savefig(output_prefix.with_suffix(".svg"))
 
     fig, ax = plt.subplots(figsize=(5, 3), layout="constrained")
     values = [x[0] for x in sorted_categories]
@@ -373,7 +373,7 @@ def _main(input_dir: Path, output_prefix: Path):
         values, labels=labels, colors=colors, hatch=hatches, textprops={"fontsize": 6}
     )
 
-    fig.savefig(output_prefix.with_suffix(".pie.png"), dpi=200)
+    fig.savefig(output_prefix.with_suffix(".pie.svg"), dpi=200)
 
 
 def main():

bench_runner/templates/README.md

@@ -12,12 +12,12 @@ Here are some recent and important revisions. 👉 [Complete list of results](RE
 
 `*` indicates that the exact same versions of pyperformance was not used.
 
-![Longitudinal speed improvement](/longitudinal.png)
+![Longitudinal speed improvement](/longitudinal.svg)
 
 Improvement of the geometric mean of key merged benchmarks, computed with `pyperf compare`.
 The results have a resolution of 0.01 (1%).
 
-![Configuration speed improvement](/configs.png)
+![Configuration speed improvement](/configs.svg)
 
 ## Documentation
 
@@ -45,7 +45,7 @@ When the benchmarking is complete, the results are published to this repository
 Each set of benchmarks will have:
 
 - The raw `.json` results from pyperformance.
-- Comparisons against important reference releases, as well as the merge base of the branch if `benchmark_base` was selected.  These include
+- Comparisons against important reference releases, as well as the merge base of the branch if `benchmark_base` was selected. These include
   - A markdown table produced by `pyperf compare_to`.
   - A set of "violin" plots showing the distribution of results for each benchmark.
 

bench_runner/templates/_generate.src.yml

@@ -48,5 +48,5 @@ jobs:
         uses: EndBug/add-and-commit@v9
         if: ${{ !inputs.dry_run }}
         with:
-          add: "['results', 'README.md', 'RESULTS.md', 'longitudinal.png', 'longitudinal.json', 'configs.png', 'configs.json', 'memory_long.png', 'memory_long.json', 'memory_configs.png', 'memory_configs.json']"
+          add: "['results', 'README.md', 'RESULTS.md', 'longitudinal.svg', 'longitudinal.json', 'configs.svg', 'configs.json', 'memory_long.svg', 'memory_long.json', 'memory_configs.svg', 'memory_configs.json']"
           message: Benchmarking results for @${{ github.actor }}

bench_runner/util.py

@@ -19,6 +19,6 @@ def get_benchmark_hash() -> str:
 
 def apply_suffix(path: Path, suffix: str) -> Path:
     """
-    Like Path.with_suffix but allows adding things like "-mem.png".
+    Like Path.with_suffix but allows adding things like "-mem.svg".
     """
     return path.parent / (path.stem + suffix)

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "rich==13.7.1",
     "rich-argparse==1.5.2",
     "ruamel.yaml==0.18.6",
+    "scour==0.38.2",
     "wheel",
 ]
 dynamic = ["version"]

tests/test_compare.py

@@ -27,7 +27,7 @@ def test_compare_1_to_n(tmp_path, monkeypatch):
     assert len(files) == 7
     expected_files = {"README.md"}
     for hash in hashes[1:]:
-        for suffix in [".md", ".png", "-mem.png"]:
+        for suffix in [".md", ".svg", "-mem.svg"]:
             expected_files.add(f"linux-{hash}-vs-{hashes[0]}{suffix}")
     assert set(x.name for x in files) == expected_files
 
@@ -38,8 +38,8 @@ def test_compare_1_to_n(tmp_path, monkeypatch):
 
         | commit | change |
         | -- | -- |
-        | eb0004c (eb0004c) | 1.57x ↑[📄](linux-eb0004c-vs-9d38120.md)[📈](linux-eb0004c-vs-9d38120.png)[🧠](linux-eb0004c-vs-9d38120-mem.png) |
-        | b0e1f9c (b0e1f9c) | 1.70x ↑[📄](linux-b0e1f9c-vs-9d38120.md)[📈](linux-b0e1f9c-vs-9d38120.png)[🧠](linux-b0e1f9c-vs-9d38120-mem.png) |
+        | eb0004c (eb0004c) | 1.57x ↑[📄](linux-eb0004c-vs-9d38120.md)[📈](linux-eb0004c-vs-9d38120.svg)[🧠](linux-eb0004c-vs-9d38120-mem.svg) |
+        | b0e1f9c (b0e1f9c) | 1.70x ↑[📄](linux-b0e1f9c-vs-9d38120.md)[📈](linux-b0e1f9c-vs-9d38120.svg)[🧠](linux-b0e1f9c-vs-9d38120-mem.svg) |
         """  # noqa
     ).strip()
     assert expected in content
@@ -61,7 +61,7 @@ def test_compare_n_to_n(tmp_path, monkeypatch):
         for hash2 in hashes:
             if hash1 == hash2:
                 continue
-            for suffix in [".md", ".png", "-mem.png"]:
+            for suffix in [".md", ".svg", "-mem.svg"]:
                 expected_files.add(f"linux-{hash2}-vs-{hash1}{suffix}")
     assert set(x.name for x in files) == expected_files
 
@@ -72,9 +72,9 @@ def test_compare_n_to_n(tmp_path, monkeypatch):
 
         |  | 9d38120 (9d38120) | eb0004c (eb0004c) | b0e1f9c (b0e1f9c) |
         | -- | -- | -- | -- |
-        | 9d38120 |  | 1.57x ↑[📄](linux-eb0004c-vs-9d38120.md)[📈](linux-eb0004c-vs-9d38120.png)[🧠](linux-eb0004c-vs-9d38120-mem.png) | 1.70x ↑[📄](linux-b0e1f9c-vs-9d38120.md)[📈](linux-b0e1f9c-vs-9d38120.png)[🧠](linux-b0e1f9c-vs-9d38120-mem.png) |
-        | eb0004c | 1.57x ↓[📄](linux-9d38120-vs-eb0004c.md)[📈](linux-9d38120-vs-eb0004c.png)[🧠](linux-9d38120-vs-eb0004c-mem.png) |  | 1.08x ↑[📄](linux-b0e1f9c-vs-eb0004c.md)[📈](linux-b0e1f9c-vs-eb0004c.png)[🧠](linux-b0e1f9c-vs-eb0004c-mem.png) |
-        | b0e1f9c | 1.70x ↓[📄](linux-9d38120-vs-b0e1f9c.md)[📈](linux-9d38120-vs-b0e1f9c.png)[🧠](linux-9d38120-vs-b0e1f9c-mem.png) | 1.08x ↓[📄](linux-eb0004c-vs-b0e1f9c.md)[📈](linux-eb0004c-vs-b0e1f9c.png)[🧠](linux-eb0004c-vs-b0e1f9c-mem.png) |  |
+        | 9d38120 |  | 1.57x ↑[📄](linux-eb0004c-vs-9d38120.md)[📈](linux-eb0004c-vs-9d38120.svg)[🧠](linux-eb0004c-vs-9d38120-mem.svg) | 1.70x ↑[📄](linux-b0e1f9c-vs-9d38120.md)[📈](linux-b0e1f9c-vs-9d38120.svg)[🧠](linux-b0e1f9c-vs-9d38120-mem.svg) |
+        | eb0004c | 1.57x ↓[📄](linux-9d38120-vs-eb0004c.md)[📈](linux-9d38120-vs-eb0004c.svg)[🧠](linux-9d38120-vs-eb0004c-mem.svg) |  | 1.08x ↑[📄](linux-b0e1f9c-vs-eb0004c.md)[📈](linux-b0e1f9c-vs-eb0004c.svg)[🧠](linux-b0e1f9c-vs-eb0004c-mem.svg) |
+        | b0e1f9c | 1.70x ↓[📄](linux-9d38120-vs-b0e1f9c.md)[📈](linux-9d38120-vs-b0e1f9c.svg)[🧠](linux-9d38120-vs-b0e1f9c-mem.svg) | 1.08x ↓[📄](linux-eb0004c-vs-b0e1f9c.md)[📈](linux-eb0004c-vs-b0e1f9c.svg)[🧠](linux-eb0004c-vs-b0e1f9c-mem.svg) |  |
         """  # noqa
     ).strip()
     assert expected in content

tests/test_generate_results.py

@@ -35,13 +35,13 @@ def _run_for_bases(bases, repo_path, force=False, has_base=[], check_readmes=Tru
 
         if any(base in dirpath.name for base in has_base):
             assert files_by_type[".md"] == 4
-            assert files_by_type[".png"] == 4
+            assert files_by_type[".svg"] == 4
         elif any(base in dirpath.name for base in bases):
             assert files_by_type[".md"] in (2, 3)
-            assert files_by_type[".png"] in (1, 2)
+            assert files_by_type[".svg"] in (1, 2)
         else:
             assert files_by_type[".md"] in (3, 4)
-            assert files_by_type[".png"] in (2, 3)
+            assert files_by_type[".svg"] in (2, 3)
 
         # Make sure all files in the directory have a link
         contents = (dirpath / "README.md").read_text()

tests/test_result.py

@@ -14,12 +14,14 @@
 
 def _copy_results(tmp_path):
     results_path = tmp_path / "results"
+    shutil.copyfile(DATA_PATH / "bases.txt", tmp_path / "bases.txt")
     shutil.copytree(DATA_PATH / "results", tmp_path / "results")
     return results_path
 
 
-def test_load_all_results(tmp_path):
+def test_load_all_results(tmp_path, monkeypatch):
     results_path = _copy_results(tmp_path)
+    monkeypatch.chdir(tmp_path)
 
     results = mod_result.load_all_results(["3.10.4", "3.11.0b3"], results_path)
 

tests/test_run_benchmarks.py

@@ -147,8 +147,9 @@ def test_run_benchmarks(benchmarks_checkout):
     assert returncode == 1
 
 
-def test_should_run_exists_noforce(benchmarks_checkout, capsys):
+def test_should_run_exists_noforce(benchmarks_checkout, capsys, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
+    monkeypatch.chdir(repo)
 
     should_run._main(
         False,
@@ -166,8 +167,9 @@ def test_should_run_exists_noforce(benchmarks_checkout, capsys):
     assert (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
 
 
-def test_should_run_diff_machine_noforce(benchmarks_checkout, capsys):
+def test_should_run_diff_machine_noforce(benchmarks_checkout, capsys, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
+    monkeypatch.chdir(repo)
 
     should_run._main(
         False,
@@ -185,8 +187,9 @@ def test_should_run_diff_machine_noforce(benchmarks_checkout, capsys):
     assert len(list((repo / "results" / "bm-20220323-3.10.4-9d38120").iterdir())) == 1
 
 
-def test_should_run_all_noforce(benchmarks_checkout, capsys):
+def test_should_run_all_noforce(benchmarks_checkout, capsys, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
+    monkeypatch.chdir(repo)
 
     should_run._main(
         False,
@@ -204,8 +207,9 @@ def test_should_run_all_noforce(benchmarks_checkout, capsys):
     assert len(list((repo / "results" / "bm-20220323-3.10.4-9d38120").iterdir())) == 1
 
 
-def test_should_run_noexists_noforce(benchmarks_checkout, capsys):
+def test_should_run_noexists_noforce(benchmarks_checkout, capsys, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
+    monkeypatch.chdir(repo)
     shutil.rmtree(repo / "results" / "bm-20220323-3.10.4-9d38120")
 
     should_run._main(
@@ -226,6 +230,7 @@ def test_should_run_noexists_noforce(benchmarks_checkout, capsys):
 
 def test_should_run_exists_force(benchmarks_checkout, capsys, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
+    monkeypatch.chdir(repo)
 
     removed_paths = []
 
@@ -253,14 +258,15 @@ def remove(repo, path):
     assert captured.out.splitlines()[-1].strip() == "should_run=true"
     assert (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
     assert set(x.name for x in removed_paths) == {
-        "bm-20220323-linux-x86_64-python-main-3.10.4-9d38120-vs-3.11.0b3.png",
+        "bm-20220323-linux-x86_64-python-main-3.10.4-9d38120-vs-3.11.0b3.svg",
         "README.md",
         "bm-20220323-linux-x86_64-python-main-3.10.4-9d38120-vs-3.11.0b3.md",
     }
 
 
-def test_should_run_noexists_force(benchmarks_checkout, capsys):
+def test_should_run_noexists_force(benchmarks_checkout, capsys, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
+    monkeypatch.chdir(repo)
     shutil.rmtree(repo / "results" / "bm-20220323-3.10.4-9d38120")
 
     should_run._main(
@@ -279,8 +285,9 @@ def test_should_run_noexists_force(benchmarks_checkout, capsys):
     assert not (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
 
 
-def test_should_run_checkout_failed(tmp_path, capsys):
+def test_should_run_checkout_failed(tmp_path, capsys, monkeypatch):
     repo = _copy_repo(tmp_path)
+    monkeypatch.chdir(repo)
     cpython_path = tmp_path / "cpython"
     cpython_path.mkdir()
     subprocess.check_call(["git", "init"], cwd=cpython_path)