Minor improvements to benchmarking setup (#5843)

scripts/benchmark/pyproject.toml

@@ -2,7 +2,7 @@
 name = "benchmark"
 version = "0.0.1"
 description = "Benchmark package resolution tools"
-requires-python = ">=3.12"
+requires-python = ">=3.11"
 dependencies = [
   "pdm",
   "pip-tools",

scripts/benchmark/src/benchmark/__init__.py

@@ -21,12 +21,15 @@ class Hyperfine(typing.NamedTuple):
     commands: list[Command]
     """The commands to benchmark."""
 
-    warmup: int
+    warmup: int | None
     """The number of warmup runs to perform."""
 
-    min_runs: int
+    min_runs: int | None
     """The minimum number of runs to perform."""
 
+    runs: int | None
+    """The number of runs to perform."""
+
     verbose: bool
     """Whether to print verbose output."""
 
@@ -45,10 +48,15 @@ def run(self) -> None:
         # Preamble: benchmark-wide setup.
         if self.verbose:
             args.append("--show-output")
-        args.append("--warmup")
-        args.append(str(self.warmup))
-        args.append("--min-runs")
-        args.append(str(self.min_runs))
+        if self.warmup is not None:
+            args.append("--warmup")
+            args.append(str(self.warmup))
+        if self.min_runs is not None:
+            args.append("--min-runs")
+            args.append(str(self.min_runs))
+        if self.runs is not None:
+            args.append("--runs")
+            args.append(str(self.runs))
 
         # Add all command names,
         for command in self.commands:

scripts/benchmark/src/benchmark/resolver.py

@@ -28,21 +28,21 @@
 is provided, and the installation benchmarks when a `requirements.txt` file is provided:
 
     # Run the resolution benchmarks against the Trio project.
-    uv run benchmark\
+    uv run resolver \
         --uv-path ../../target/release/uv \
         --uv-path ../../target/release/baseline \
         ../requirements/trio.in
 
     # Run the installation benchmarks against the Trio project.
-    uv run benchmark\
+    uv run resolver \
         --uv-path ../../target/release/uv \
         --uv-path ../../target/release/baseline \
         ../requirements/compiled/trio.txt
 
 You can also specify the benchmark to run explicitly:
 
     # Run the "uncached install" benchmark against the Trio project.
-    uv run benchmark\
+    uv run resolver \
         --uv-path ../../target/release/uv \
         --uv-path ../../target/release/baseline \
         --benchmark install-cold \
@@ -302,7 +302,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
-            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
+            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12.3 {venv_dir}",
             command=[
                 self.path,
                 os.path.abspath(requirements_file),
@@ -319,7 +319,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
-            prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
+            prepare=f"virtualenv --clear -p 3.12.3 {venv_dir}",
             command=[
                 self.path,
                 os.path.abspath(requirements_file),
@@ -559,7 +559,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
                 f"rm -rf {config_dir} && "
                 f"rm -rf {cache_dir} && "
                 f"rm -rf {data_dir} &&"
-                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
             ),
             command=[
                 f"POETRY_CONFIG_DIR={config_dir}",
@@ -599,7 +599,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
-            prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
+            prepare=f"virtualenv --clear -p 3.12.3 {venv_dir}",
             command=[
                 f"POETRY_CONFIG_DIR={config_dir}",
                 f"POETRY_CACHE_DIR={cache_dir}",
@@ -636,7 +636,7 @@ def setup(self, requirements_file: str, *, cwd: str) -> None:
 
         # Create a PDM project.
         subprocess.check_call(
-            [self.path, "init", "--non-interactive", "--python", "3.12"],
+            [self.path, "init", "--non-interactive", "--python", "3.12.3"],
             cwd=cwd,
             stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL,
@@ -792,7 +792,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
             prepare=(
                 f"rm -rf {cache_dir} && "
                 f"{self.path} config cache_dir {cache_dir} && "
-                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
             ),
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
@@ -826,7 +826,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
             prepare=(
                 f"{self.path} config cache_dir {cache_dir} && "
-                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
             ),
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
@@ -841,7 +841,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 class UvPip(Suite):
     def __init__(self, *, path: str | None = None) -> Command | None:
         """Initialize a uv benchmark."""
-        self.name = path or "uv"
+        self.name = path or "uv pip"
         self.path = path or os.path.join(
             os.path.dirname(
                 os.path.dirname(
@@ -967,7 +967,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
-            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
+            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12.3 {venv_dir}",
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
                 self.path,
@@ -985,7 +985,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
-            prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
+            prepare=f"virtualenv --clear -p 3.12.3 {venv_dir}",
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
                 self.path,
@@ -1207,7 +1207,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
             name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
             prepare=(
                 f"rm -rf {cache_dir} && "
-                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
             ),
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
@@ -1243,7 +1243,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
-            prepare=(f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"),
+            prepare=(f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"),
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
                 self.path,
@@ -1287,6 +1287,11 @@ def main():
         help="The minimum number of runs to perform.",
         default=10,
     )
+    parser.add_argument(
+        "--runs",
+        type=int,
+        help="The number of runs to perform.",
+    )
     parser.add_argument(
         "--benchmark",
         "-b",
@@ -1373,6 +1378,7 @@ def main():
     json = args.json
     warmup = args.warmup
     min_runs = args.min_runs
+    runs = args.runs
 
     requirements_file = os.path.abspath(args.file)
     if not os.path.exists(requirements_file):
@@ -1411,6 +1417,7 @@ def main():
             PipSync(),
             PipCompile(),
             Poetry(),
+            Pdm(),
             UvPip(),
             UvProject(),
         ]
@@ -1453,6 +1460,7 @@ def main():
                     commands=commands,
                     warmup=warmup,
                     min_runs=min_runs,
+                    runs=runs,
                     verbose=verbose,
                     json=json,
                 )

scripts/benchmark/src/benchmark/tools.py

@@ -245,6 +245,11 @@ def main():
         help="The minimum number of runs to perform.",
         default=10,
     )
+    parser.add_argument(
+        "--runs",
+        type=int,
+        help="The number of runs to perform.",
+    )
     parser.add_argument(
         "--benchmark",
         "-b",
@@ -287,6 +292,7 @@ def main():
     json = args.json
     warmup = args.warmup
     min_runs = args.min_runs
+    runs = args.runs
 
     # Determine the tools to benchmark, based on the user-provided arguments.
     suites = []
@@ -328,6 +334,7 @@ def main():
                     commands=commands,
                     warmup=warmup,
                     min_runs=min_runs,
+                    runs=runs,
                     verbose=verbose,
                     json=json,
                 )