Skip to content

Commit

Permalink
add benchmark
Browse files Browse the repository at this point in the history
  • Loading branch information
@vincentsarago
vincentsarago committed Oct 17, 2024
1 parent 891cf23 commit fc93540
Show file tree
Hide file tree
Showing 7 changed files with 254 additions and 0 deletions.
35 changes: 35 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,41 @@ jobs:
name: Python${{ matrix.python-version }}
fail_ci_if_error: false

benchmark:
needs: [tests]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e ".[benchmark]"
- name: Run Benchmark
run: |
python -m pytest tests/benchmarks.py -k morecantile --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min' --benchmark-json output.json
- name: Store and Compare benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
name: morecantile Benchmarks
tool: 'pytest'
output-file-path: output.json
alert-threshold: '150%'
comment-on-alert: true
fail-on-alert: false
# GitHub API token to make a commit comment
github-token: ${{ secrets.GITHUB_TOKEN }}
gh-pages-branch: 'gh-benchmarks'
# Make a commit on `gh-pages` only if main
auto-push: ${{ github.ref == 'refs/heads/main' }}
benchmark-data-dir-path: dev/benchmarks

publish:
needs: [tests]
runs-on: ubuntu-latest
Expand Down
7 changes: 7 additions & 0 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ This repo is set to use `pre-commit` to run *isort*, *flake8*, *pydocstring*, *b
$ pre-commit install
```

##### Performance tests

```sh
python -m pip install -e ".[benchmark]"
python -m pytest tests/benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min'
```

### Docs

```bash
Expand Down
63 changes: 63 additions & 0 deletions benchmarks/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@

## Benchmark

Compare `mercantile`, `utiles` and `morecantile`

```sh
python -m pip install -r requirements.txt
python -m pytest benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min'
```

```
----------------------------------------------------- benchmark 'bounds': 21 tests ----------------------------------------------------
Name (time in ns) Min Max Mean Median
---------------------------------------------------------------------------------------------------------------------------------------
test_bounds[utiles-(486, 332, 30)] 83.0000 (1.0) 52,000.0000 (52.97) 195.3371 (1.48) 208.0000 (1.60)
test_bounds[utiles-(0, 0, 0)] 83.0000 (1.00) 31,167.0000 (31.75) 193.5957 (1.47) 208.0000 (1.60)
test_bounds[utiles-(1, 0, 1)] 125.0000 (1.51) 981.6700 (1.0) 132.0177 (1.0) 130.8300 (1.01)
test_bounds[utiles-(1, 40, 7)] 125.0000 (1.51) 8,606.2500 (8.77) 132.5298 (1.00) 130.0000 (1.0)
test_bounds[utiles-(486, 332, 10)] 128.3300 (1.55) 1,600.8300 (1.63) 133.4046 (1.01) 131.6700 (1.01)
test_bounds[utiles-(1, 1, 1)] 129.5900 (1.56) 16,846.2500 (17.16) 141.9846 (1.08) 134.1700 (1.03)
test_bounds[utiles-(486, 332, 20)] 138.3300 (1.67) 1,123.3300 (1.14) 142.5420 (1.08) 141.2500 (1.09)
test_bounds[mercantile-(1, 40, 7)] 1,000.0000 (12.05) 92,416.0000 (94.14) 1,193.0112 (9.04) 1,167.0000 (8.98)
test_bounds[mercantile-(1, 1, 1)] 1,075.0000 (12.95) 48,066.8000 (48.96) 1,154.8100 (8.75) 1,141.6000 (8.78)
test_bounds[mercantile-(0, 0, 0)] 1,083.0000 (13.05) 21,041.0000 (21.43) 1,214.6049 (9.20) 1,208.0000 (9.29)
test_bounds[mercantile-(1, 0, 1)] 1,083.0000 (13.05) 75,291.0000 (76.70) 1,213.7837 (9.19) 1,208.0000 (9.29)
test_bounds[mercantile-(486, 332, 10)] 1,083.0000 (13.05) 55,750.0000 (56.79) 1,223.7598 (9.27) 1,208.0000 (9.29)
test_bounds[mercantile-(486, 332, 20)] 1,083.0000 (13.05) 89,041.0000 (90.70) 1,248.4256 (9.46) 1,209.0000 (9.30)
test_bounds[mercantile-(486, 332, 30)] 1,083.0000 (13.05) 194,958.0000 (198.60) 1,239.4339 (9.39) 1,209.0000 (9.30)
test_bounds[morecantile-(1, 0, 1)] 20,083.0000 (241.96) 132,625.0000 (135.10) 20,893.1235 (158.26) 20,667.0000 (158.98)
test_bounds[morecantile-(0, 0, 0)] 20,166.0000 (242.96) 120,333.0000 (122.58) 20,974.1961 (158.87) 20,708.0000 (159.29)
test_bounds[morecantile-(1, 1, 1)] 20,334.0000 (244.99) 60,958.0000 (62.10) 21,236.2953 (160.86) 20,750.0000 (159.62)
test_bounds[morecantile-(1, 40, 7)] 21,667.0000 (261.05) 173,458.0000 (176.70) 22,622.6537 (171.36) 22,333.0000 (171.79)
test_bounds[morecantile-(486, 332, 10)] 22,166.0000 (267.06) 205,125.0000 (208.96) 23,512.1607 (178.10) 22,875.0000 (175.96)
test_bounds[morecantile-(486, 332, 20)] 24,708.0000 (297.69) 129,250.0000 (131.66) 25,775.0796 (195.24) 25,375.0000 (195.19)
test_bounds[morecantile-(486, 332, 30)] 71,292.0000 (858.94) 205,625.0000 (209.46) 73,569.4565 (557.27) 72,667.0000 (558.98)
---------------------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------- benchmark 'xy_bounds': 21 tests -----------------------------------------------------
Name (time in ns) Min Max Mean Median
------------------------------------------------------------------------------------------------------------------------------------------
test_xy_bounds[utiles-(0, 0, 0)] 83.0000 (1.0) 57,375.0000 (112.55) 145.9127 (1.57) 125.0000 (1.36)
test_xy_bounds[utiles-(1, 1, 1)] 88.3400 (1.06) 1,335.4200 (2.62) 93.1478 (1.00) 92.5000 (1.00)
test_xy_bounds[utiles-(1, 40, 7)] 88.7500 (1.07) 1,451.2500 (2.85) 93.1258 (1.0) 92.0900 (1.0)
test_xy_bounds[utiles-(1, 0, 1)] 89.5900 (1.08) 861.6600 (1.69) 93.6428 (1.01) 93.3300 (1.01)
test_xy_bounds[utiles-(486, 332, 10)] 90.0000 (1.08) 509.7950 (1.0) 93.2818 (1.00) 92.9150 (1.01)
test_xy_bounds[utiles-(486, 332, 20)] 90.4100 (1.09) 864.1700 (1.70) 94.2974 (1.01) 94.1600 (1.02)
test_xy_bounds[utiles-(486, 332, 30)] 90.4100 (1.09) 1,377.5000 (2.70) 94.7366 (1.02) 94.1600 (1.02)
test_xy_bounds[mercantile-(0, 0, 0)] 708.0000 (8.53) 42,250.0000 (82.88) 843.2292 (9.05) 833.0000 (9.05)
test_xy_bounds[mercantile-(486, 332, 10)] 731.2500 (8.81) 5,393.7500 (10.58) 770.8331 (8.28) 764.6000 (8.30)
test_xy_bounds[mercantile-(486, 332, 30)] 731.2500 (8.81) 7,316.6500 (14.35) 770.2423 (8.27) 764.6000 (8.30)
test_xy_bounds[mercantile-(1, 0, 1)] 732.1429 (8.82) 9,404.8571 (18.45) 791.7721 (8.50) 785.7143 (8.53)
test_xy_bounds[mercantile-(1, 40, 7)] 733.3000 (8.83) 5,950.0000 (11.67) 772.3273 (8.29) 766.7000 (8.33)
test_xy_bounds[mercantile-(486, 332, 20)] 733.3500 (8.84) 7,408.3000 (14.53) 773.4718 (8.31) 768.7500 (8.35)
test_xy_bounds[mercantile-(1, 1, 1)] 735.4000 (8.86) 4,762.5000 (9.34) 783.2100 (8.41) 777.1000 (8.44)
test_xy_bounds[morecantile-(0, 0, 0)] 3,042.0000 (36.65) 108,208.0000 (212.26) 3,287.1089 (35.30) 3,250.0000 (35.29)
test_xy_bounds[morecantile-(1, 0, 1)] 3,291.0000 (39.65) 68,166.0000 (133.71) 3,503.4695 (37.62) 3,459.0000 (37.56)
test_xy_bounds[morecantile-(1, 1, 1)] 3,291.0000 (39.65) 72,875.0000 (142.95) 3,506.8544 (37.66) 3,500.0000 (38.01)
test_xy_bounds[morecantile-(1, 40, 7)] 4,375.0000 (52.71) 76,375.0000 (149.82) 4,630.2893 (49.72) 4,584.0000 (49.78)
test_xy_bounds[morecantile-(486, 332, 10)] 4,875.0000 (58.73) 85,208.0000 (167.14) 5,197.3581 (55.81) 5,125.0000 (55.65)
test_xy_bounds[morecantile-(486, 332, 20)] 6,916.0000 (83.33) 78,458.0000 (153.90) 7,221.0669 (77.54) 7,166.0000 (77.82)
test_xy_bounds[morecantile-(486, 332, 30)] 52,667.0000 (634.54) 404,208.0000 (792.88) 54,749.1232 (587.90) 54,333.0000 (590.00)
------------------------------------------------------------------------------------------------------------------------------------------
```
96 changes: 96 additions & 0 deletions benchmarks/benchmarks.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
"""Morecantile/Mercantile/Utiles comparison benchmark
The benchmark suite is adapted from jessekrubin/utiles
https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py#L17-L25
"""
# This file is a modified version of https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py.
#
# The original license follows.
#
# MIT License
#
# Copyright (c) 2023 jessekrubin
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from typing import Callable, Tuple

import mercantile
import pytest
import utiles

import morecantile

tms = morecantile.tms.get("WebMercatorQuad")

TEST_TILES = (
(0, 0, 0),
(1, 0, 1),
(1, 1, 1),
(1, 40, 7),
(486, 332, 10),
# HIGH ZOOM
(486, 332, 20),
# OUTSIDE TMS Range
(486, 332, 30),
)


@pytest.mark.parametrize(
"tile",
[pytest.param(t, id=str(t)) for t in TEST_TILES],
)
@pytest.mark.parametrize(
"func",
[
pytest.param(mercantile.bounds, id="mercantile"),
pytest.param(tms.bounds, id="morecantile"),
pytest.param(utiles.bounds, id="utiles"),
],
)
@pytest.mark.benchmark(group="bounds")
def test_bounds(
tile: Tuple[int, int, int],
func: Callable[[Tuple[int, int, int]], Tuple[float, float]],
benchmark,
) -> None:
"""Benchmark bounds() method."""
_ = benchmark(func, *tile)


@pytest.mark.parametrize(
"tile",
[pytest.param(t, id=str(t)) for t in TEST_TILES],
)
@pytest.mark.parametrize(
"func",
[
pytest.param(mercantile.xy_bounds, id="mercantile"),
pytest.param(tms.xy_bounds, id="morecantile"),
pytest.param(utiles.xy_bounds, id="utiles"),
],
)
@pytest.mark.benchmark(group="xy_bounds")
def test_xy_bounds(
tile: Tuple[int, int, int],
func: Callable[[Tuple[int, int, int]], Tuple[float, float]],
benchmark,
) -> None:
"""Benchmark xy_bounds() method."""
_ = benchmark(func, *tile)
6 changes: 6 additions & 0 deletions benchmarks/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pytest
pytest-benchmark

morecantile
mercantile
utiles
4 changes: 4 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ test = [
"pytest-cov",
"rasterio>=1.2.1",
]
benchmark = [
"pytest",
"pytest-benchmark",
]
dev = [
"pre-commit",
"bump-my-version",
Expand Down
43 changes: 43 additions & 0 deletions tests/benchmarks.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
"""Morecantile benchmark."""

import pytest

import morecantile
from morecantile.commons import BoundingBox

tms = morecantile.tms.get("WebMercatorQuad")

# Test tiles from https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py
TEST_TILES = (
(0, 0, 0),
(1, 0, 1),
(1, 1, 1),
(1, 40, 7),
(486, 332, 10),
# HIGH ZOOM
(486, 332, 20),
# OUTSIDE TMS Range
(486, 332, 30),
)


@pytest.mark.parametrize("tile", TEST_TILES)
def test_bounds(tile, benchmark):
str_tile = "Tile(x={},y={},z={})".format(*tile)
benchmark.name = f"morecantile.bounds-{str_tile}"
benchmark.fullname = f"morecantile.bounds-{str_tile}"
benchmark.group = "morecantile.bounds"

r = benchmark(tms.bounds, *tile)
assert isinstance(r, BoundingBox)


@pytest.mark.parametrize("tile", TEST_TILES)
def test_xy_bounds(tile, benchmark) -> None:
str_tile = "Tile(x={},y={},z={})".format(*tile)
benchmark.name = f"morecantile.xy_bounds-{str_tile}"
benchmark.fullname = f"morecantile.xy_bounds-{str_tile}"
benchmark.group = "morecantile.xy_bounds"

r = benchmark(tms.xy_bounds, *tile)
assert isinstance(r, BoundingBox)

0 comments on commit fc93540

Please sign in to comment.