diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e265082..02fe0e9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,7 +6,6 @@ build: python: "3.11" python: - system_packages: false install: - method: pip path: . diff --git a/docs/configuration.rst b/docs/configuration.rst index 9e6914f..52c4272 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -84,11 +84,11 @@ Directory containing baseline images ------------------------------------ | **kwarg**: ``baseline_dir=`` | **CLI**: ``--mpl-baseline-path=`` -| **INI**: --- +| **INI**: ``mpl-baseline-path`` | Default: ``baseline/`` *(relative to the test file)* The directory containing the baseline images that will be compared to the test figures. -The kwarg option (``baseline_dir``) is relative to the test file, while the CLI option (``--mpl-baseline-path``) is relative to where pytest was run. +The kwarg option (``baseline_dir``) is relative to the test file, while the CLI option (``--mpl-baseline-path``) and INI option (``mpl-baseline-path``) are relative to where pytest was run. Absolute paths can also be used. If the directory does not exist, it will be created along with any missing parent directories. @@ -160,7 +160,7 @@ If you specify a filename that has an extension other than ``png``, you must als Whether to include the module name in the filename -------------------------------------------------- | **kwarg**: --- -| **CLI**: --- +| **CLI**: ``--mpl-use-full-test-name`` | **INI**: ``mpl-use-full-test-name`` | Default: ``False`` @@ -191,15 +191,19 @@ File containing baseline hashes ------------------------------- | **kwarg**: ``hash_library=`` | **CLI**: ``--mpl-hash-library=`` -| **INI**: --- +| **INI**: ``mpl-hash-library = `` | Default: *no hash comparison* The file containing the baseline hashes that will be compared to the test figures. -Both the kwarg option (``hash_library``) and the CLI option (``--mpl-hash-library``) are relative to the test file. -In this case, the CLI option takes precedence over the kwarg option. +The kwarg option (``hash_library``) is relative to the test file, while the INI option (``mpl-hash-library``) is relative to where pytest was run. The file must be a JSON file in the same format as one generated by ``--mpl-generate-hash-library``. If its directory does not exist, it will be created along with any missing parent directories. +.. attention:: + + For backwards compatibility, the CLI option (``--mpl-hash-library``) is relative to the test file. + Also, the CLI option takes precedence over the kwarg option, but the kwarg option takes precedence over the INI option as usual. + Configuring this option disables baseline image comparison. If you want to enable both hash and baseline image comparison, which we call :doc:`"hybrid mode" `, you must explicitly set the :ref:`baseline directory configuration option `. @@ -222,8 +226,8 @@ Adjusting these options *may* allow tests to pass across a range of Matplotlib a RMS tolerance ------------- | **kwarg**: ``tolerance=`` -| **CLI**: --- -| **INI**: --- +| **CLI**: ``--mpl-default-tolerance=`` +| **INI**: ``mpl-default-tolerance = `` | Default: ``2`` The maximum RMS difference between the result image and the baseline image before the test fails. @@ -306,8 +310,8 @@ A dictionary of keyword arguments to pass to :func:`matplotlib.pyplot.savefig`. Matplotlib style ---------------- | **kwarg**: ``style=`` -| **CLI**: --- -| **INI**: --- +| **CLI**: ``--mpl-default-style=`` +| **INI**: ``mpl-default-style = `` | Default: ``"classic"`` The Matplotlib style to use when saving the figure. @@ -336,8 +340,8 @@ See the :func:`matplotlib.style.context` ``style`` documentation for the options Matplotlib backend ------------------ | **kwarg**: ``backend=`` -| **CLI**: --- -| **INI**: --- +| **CLI**: ``--mpl-default-backend=`` +| **INI**: ``mpl-default-backend = `` | Default: ``"agg"`` The Matplotlib backend to use when saving the figure. @@ -395,7 +399,7 @@ Generate test summaries ----------------------- | **kwarg**: --- | **CLI**: ``--mpl-generate-summary={html,json,basic-html}`` -| **INI**: --- +| **INI**: ``mpl-generate-summary = {html,json,basic-html}`` | Default: ``None`` This option specifies the format of the test summary report to generate, if any. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index afd38f2..8a96be0 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -44,7 +44,11 @@ from pytest_mpl.summary.html import generate_summary_basic_html, generate_summary_html -SUPPORTED_FORMATS = {'html', 'json', 'basic-html'} +DEFAULT_STYLE = "classic" +DEFAULT_TOLERANCE = 2 +DEFAULT_BACKEND = "agg" + +SUPPORTED_FORMATS = {"html", "json", "basic-html"} SHAPE_MISMATCH_ERROR = """Error: Image dimensions did not match. Expected shape: {expected_shape} @@ -125,71 +129,119 @@ def pytest_report_header(config, startdir): def pytest_addoption(parser): group = parser.getgroup("matplotlib image comparison") - group.addoption('--mpl', action='store_true', - help="Enable comparison of matplotlib figures to reference files") - group.addoption('--mpl-generate-path', - help="directory to generate reference images in, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-generate-hash-library', - help="filepath to save a generated hash library, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-baseline-path', - help="directory containing baseline images, relative to " - "location where py.test is run unless --mpl-baseline-relative is given. " - "This can also be a URL or a set of comma-separated URLs (in case " - "mirrors are specified)", action='store') - group.addoption("--mpl-baseline-relative", help="interpret the baseline directory as " - "relative to the test location.", action="store_true") - group.addoption('--mpl-hash-library', - help="json library of image hashes, relative to " - "location where py.test is run", action='store') - group.addoption('--mpl-generate-summary', action='store', - help="Generate a summary report of any failed tests" - ", in --mpl-results-path. The type of the report should be " - "specified. Supported types are `html`, `json` and `basic-html`. " - "Multiple types can be specified separated by commas.") - - results_path_help = "directory for test results, relative to location where py.test is run" - group.addoption('--mpl-results-path', help=results_path_help, action='store') - parser.addini('mpl-results-path', help=results_path_help) - - results_always_help = ("Always compare to baseline images and save result images, even for passing tests. " - "This option is automatically applied when generating a HTML summary.") - group.addoption('--mpl-results-always', action='store_true', - help=results_always_help) - parser.addini('mpl-results-always', help=results_always_help) - - parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.", - type='bool') + + msg = "Enable comparison of matplotlib figures to reference files" + group.addoption("--mpl", help=msg, action="store_true") + + msg = "directory to generate reference images in, relative to location where py.test is run" + group.addoption("--mpl-generate-path", help=msg, action="store") + + msg = "filepath to save a generated hash library, relative to location where py.test is run" + group.addoption("--mpl-generate-hash-library", help=msg, action="store") + + msg = ( + "directory containing baseline images, relative to " + "location where py.test is run unless --mpl-baseline-relative is given. " + "This can also be a URL or a set of comma-separated URLs (in case " + "mirrors are specified)" + ) + option = "mpl-baseline-path" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "interpret the baseline directory as relative to the test location." + group.addoption("--mpl-baseline-relative", help=msg, action="store_true") + + msg = "json library of image hashes, relative to location where py.test is run" + option = "mpl-hash-library" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = ( + "Generate a summary report of any failed tests" + ", in --mpl-results-path. The type of the report should be " + "specified. Supported types are `html`, `json` and `basic-html`. " + "Multiple types can be specified separated by commas." + ) + option = "mpl-generate-summary" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "directory for test results, relative to location where py.test is run" + option = "mpl-results-path" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = ( + "Always compare to baseline images and save result images, even for passing tests. " + "This option is automatically applied when generating a HTML summary." + ) + option = "mpl-results-always" + group.addoption(f"--{option}", help=msg, action="store_true") + parser.addini(option, help=msg) + + msg = "use fully qualified test name as the filename." + option = "mpl-use-full-test-name" + group.addoption(f"--{option}", help=msg, action="store_true") + parser.addini(option, help=msg, type="bool") + + msg = "default style to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-style" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "default tolerance to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-tolerance" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "default backend to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-backend" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) def pytest_configure(config): - config.addinivalue_line('markers', - "mpl_image_compare: Compares matplotlib figures " - "against a baseline image") + config.addinivalue_line( + "markers", + "mpl_image_compare: Compares matplotlib figures against a baseline image", + ) + + if ( + config.getoption("--mpl") + or config.getoption("--mpl-generate-path") is not None + or config.getoption("--mpl-generate-hash-library") is not None + ): - if (config.getoption("--mpl") or - config.getoption("--mpl-generate-path") is not None or - config.getoption("--mpl-generate-hash-library") is not None): + def get_cli_or_ini(name, default=None): + return config.getoption(f"--{name}") or config.getini(name) or default - baseline_dir = config.getoption("--mpl-baseline-path") generate_dir = config.getoption("--mpl-generate-path") generate_hash_lib = config.getoption("--mpl-generate-hash-library") - results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") - hash_library = config.getoption("--mpl-hash-library") - generate_summary = config.getoption("--mpl-generate-summary") - results_always = (config.getoption("--mpl-results-always") or - config.getini("mpl-results-always")) + baseline_dir = get_cli_or_ini("mpl-baseline-path") if config.getoption("--mpl-baseline-relative"): baseline_relative_dir = config.getoption("--mpl-baseline-path") else: baseline_relative_dir = None + use_full_test_name = get_cli_or_ini("mpl-use-full-test-name") + + hash_library = get_cli_or_ini("mpl-hash-library") + _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility + + default_tolerance = get_cli_or_ini("mpl-default-tolerance", DEFAULT_TOLERANCE) + if isinstance(default_tolerance, str): + if default_tolerance.isdigit(): # prefer int if possible + default_tolerance = int(default_tolerance) + else: + default_tolerance = float(default_tolerance) + default_style = get_cli_or_ini("mpl-default-style", DEFAULT_STYLE) + default_backend = get_cli_or_ini("mpl-default-backend", DEFAULT_BACKEND) - # Note that results_dir is an empty string if not specified - if not results_dir: - results_dir = None + results_dir = get_cli_or_ini("mpl-results-path") + results_always = get_cli_or_ini("mpl-results-always") + generate_summary = get_cli_or_ini("mpl-generate-summary") if generate_dir is not None: if baseline_dir is not None: @@ -201,19 +253,30 @@ def pytest_configure(config): baseline_dir = os.path.abspath(generate_dir) if results_dir is not None: results_dir = os.path.abspath(results_dir) - - config.pluginmanager.register(ImageComparison(config, - baseline_dir=baseline_dir, - baseline_relative_dir=baseline_relative_dir, - generate_dir=generate_dir, - results_dir=results_dir, - hash_library=hash_library, - generate_hash_library=generate_hash_lib, - generate_summary=generate_summary, - results_always=results_always)) + if hash_library is not None: + # For backwards compatibility, don't make absolute if set via CLI option + if not _hash_library_from_cli: + hash_library = os.path.abspath(hash_library) + + plugin = ImageComparison( + config, + baseline_dir=baseline_dir, + baseline_relative_dir=baseline_relative_dir, + generate_dir=generate_dir, + results_dir=results_dir, + hash_library=hash_library, + generate_hash_library=generate_hash_lib, + generate_summary=generate_summary, + results_always=results_always, + use_full_test_name=use_full_test_name, + default_style=default_style, + default_tolerance=default_tolerance, + default_backend=default_backend, + _hash_library_from_cli=_hash_library_from_cli, + ) + config.pluginmanager.register(plugin) else: - config.pluginmanager.register(FigureCloser(config)) @@ -256,24 +319,30 @@ def path_is_not_none(apath): class ImageComparison: - - def __init__(self, - config, - baseline_dir=None, - baseline_relative_dir=None, - generate_dir=None, - results_dir=None, - hash_library=None, - generate_hash_library=None, - generate_summary=None, - results_always=False - ): + def __init__( + self, + config, + baseline_dir=None, + baseline_relative_dir=None, + generate_dir=None, + results_dir=None, + hash_library=None, + generate_hash_library=None, + generate_summary=None, + results_always=False, + use_full_test_name=False, + default_style=DEFAULT_STYLE, + default_tolerance=DEFAULT_TOLERANCE, + default_backend=DEFAULT_BACKEND, + _hash_library_from_cli=False, # for backwards compatibility + ): self.config = config self.baseline_dir = baseline_dir self.baseline_relative_dir = path_is_not_none(baseline_relative_dir) self.generate_dir = path_is_not_none(generate_dir) self.results_dir = path_is_not_none(results_dir) self.hash_library = path_is_not_none(hash_library) + self._hash_library_from_cli = _hash_library_from_cli # for backwards compatibility self.generate_hash_library = path_is_not_none(generate_hash_library) if generate_summary: generate_summary = {i.lower() for i in generate_summary.split(',')} @@ -286,6 +355,11 @@ def __init__(self, results_always = True self.generate_summary = generate_summary self.results_always = results_always + self.use_full_test_name = use_full_test_name + + self.default_style = default_style + self.default_tolerance = default_tolerance + self.default_backend = default_backend # Generate the containing dir for all test results if not self.results_dir: @@ -331,7 +405,7 @@ def generate_filename(self, item): Given a pytest item, generate the figure filename. """ ext = self._file_extension(item) - if self.config.getini('mpl-use-full-test-name'): + if self.use_full_test_name: filename = generate_test_name(item) + f'.{ext}' else: compare = get_compare(item) @@ -467,7 +541,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary = {} compare = get_compare(item) - tolerance = compare.kwargs.get('tolerance', 2) + tolerance = compare.kwargs.get('tolerance', self.default_tolerance) ext = self._file_extension(item) @@ -602,7 +676,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): # Use hash library name of current test as results hash library name self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name - hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) + # Order of precedence for hash library: CLI, kwargs, INI (for backwards compatibility) + hash_library_filename = compare.kwargs.get("hash_library", None) or self.hash_library + if self._hash_library_from_cli: # for backwards compatibility + hash_library_filename = self.hash_library hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() if not Path(hash_library_filename).exists(): @@ -683,9 +760,9 @@ def pytest_runtest_call(self, item): # noqa from matplotlib.testing.decorators import ImageComparisonTest as MplImageComparisonTest remove_ticks_and_titles = MplImageComparisonTest.remove_text - style = compare.kwargs.get('style', 'classic') + style = compare.kwargs.get('style', self.default_style) remove_text = compare.kwargs.get('remove_text', False) - backend = compare.kwargs.get('backend', 'agg') + backend = compare.kwargs.get('backend', self.default_backend) ext = self._file_extension(item) diff --git a/setup.cfg b/setup.cfg index 7f9a3e3..f8026f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ filterwarnings = error ignore:distutils Version classes are deprecated ignore:the imp module is deprecated in favour of importlib + ignore:The NumPy module was reloaded [flake8] max-line-length = 100 diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..037343d --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,7 @@ +from pathlib import Path + + +def pytester_path(pytester): + if hasattr(pytester, "path"): + return pytester.path + return Path(pytester.tmpdir) # pytest v5 diff --git a/tests/test_baseline_path.py b/tests/test_baseline_path.py new file mode 100644 index 0000000..745006a --- /dev/null +++ b/tests/test_baseline_path.py @@ -0,0 +1,50 @@ +import shutil +from pathlib import Path + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected_baseline_path, success_expected", + [ + ("dir1", None, None, "dir1", True), + ("dir1", "dir2", None, "dir2", True), + ("dir1", "dir2", "dir3", "dir3", True), + ("dir1", "dir2", "dir3", "dir2", False), + (None, None, "dir3", "dir3", True), + ], +) +def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected): + path = pytester_path(pytester) + (path / expected_baseline_path).mkdir() + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + path / expected_baseline_path / "test_mpl.png", + ) + ini = f"mpl-baseline-path = {path / ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + {ini} + """ + ) + kwarg = f"baseline_dir=r'{path / kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-baseline-path={path / cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_default_backend.py b/tests/test_default_backend.py new file mode 100644 index 0000000..192d3cd --- /dev/null +++ b/tests/test_default_backend.py @@ -0,0 +1,35 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("backend1", None, None, "backend1"), + ("backend1", "backend2", None, "backend2"), + ("backend1", "backend2", "backend3", "backend3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = f"mpl-default-backend = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"backend='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-backend={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.re_match_lines([f".*(ModuleNotFound|Value)Error: .*{expected}.*"]) diff --git a/tests/test_default_style.py b/tests/test_default_style.py new file mode 100644 index 0000000..9088008 --- /dev/null +++ b/tests/test_default_style.py @@ -0,0 +1,38 @@ +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("sty1", None, None, "sty1"), + ("sty1", "sty2", None, "sty2"), + ("sty1", "sty2", "sty3", "sty3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = "mpl-default-style = " + ini if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-baseline-path = {pytester_path(pytester)} + {ini} + """ + ) + kwarg = f"style='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + + cli = "--mpl-default-style=" + cli if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*OSError: *'{expected}'*"]) diff --git a/tests/test_default_tolerance.py b/tests/test_default_tolerance.py new file mode 100644 index 0000000..93cb21b --- /dev/null +++ b/tests/test_default_tolerance.py @@ -0,0 +1,57 @@ +from pathlib import Path + +import pytest +from PIL import Image, ImageDraw + +TEST_NAME = "test_base_style" + + +@pytest.fixture(scope="module") +def baseline_image(tmpdir_factory): + path = Path(__file__).parent / "baseline" / "2.0.x" / f"{TEST_NAME}.png" + image = Image.open(path) + draw = ImageDraw.Draw(image) + draw.rectangle(((0, 0), (100, 100)), fill="red") + output = Path(tmpdir_factory.mktemp("data").join(f"{TEST_NAME}.png")) + image.save(output) + return output + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + (40, None, None, True), + (30, 40, None, True), + (30, 30, 40, True), + (30, 40, 30, False), + (40, 30, 30, False), + ], +) +def test_config(pytester, baseline_image, ini, cli, kwarg, success_expected): + ini = f"mpl-default-tolerance = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {baseline_image.parent} + {ini} + """ + ) + kwarg = f"tolerance={kwarg}" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def {TEST_NAME}(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-tolerance={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..61b1447 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,48 @@ +from helpers import pytester_path + +PYFILE = ( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ +) + + +def test_generate_baseline_images(pytester): + pytester.makepyfile(PYFILE) + baseline_dir = pytester_path(pytester) / "alternative_baseline" + result = pytester.runpytest(f"--mpl-generate-path={baseline_dir}") + result.assert_outcomes(skipped=1) + assert (baseline_dir / "test_mpl.png").exists() + + +def test_generate_baseline_hashes(pytester): + pytester.makepyfile(PYFILE) + path = pytester_path(pytester) + hash_library = path / "alternative_baseline" / "hash_library_1.json" + result = pytester.runpytest( + f"--mpl-generate-hash-library={hash_library}", + f"--mpl-results-path={path}", + ) + result.assert_outcomes(failed=1) # this option enables --mpl + assert hash_library.exists() + assert (path / "test_generate_baseline_hashes.test_mpl" / "result.png").exists() + + +def test_generate_baseline_images_and_hashes(pytester): + pytester.makepyfile(PYFILE) + path = pytester_path(pytester) + baseline_dir = path / "alternative_baseline" + hash_library = path / "alternative_baseline" / "hash_library_1.json" + result = pytester.runpytest( + f"--mpl-generate-path={baseline_dir}", + f"--mpl-generate-hash-library={hash_library}", + ) + result.assert_outcomes(skipped=1) + assert (baseline_dir / "test_mpl.png").exists() + assert hash_library.exists() diff --git a/tests/test_generate_summary.py b/tests/test_generate_summary.py new file mode 100644 index 0000000..6350975 --- /dev/null +++ b/tests/test_generate_summary.py @@ -0,0 +1,65 @@ +import json + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("json", None, {"json"}), + ("json", "html", {"html"}), + ("basic-html", "json", {"json"}), + (None, "json,basic-html,html", {"json", "basic-html", "html"}), + ], +) +def test_config(pytester, ini, cli, expected): + path = pytester_path(pytester) + ini = f"mpl-generate-summary = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-results-path = {path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-generate-summary={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + + json_summary = path / "results.json" + if "json" in expected: + with open(json_summary) as fp: + results = json.load(fp) + assert "test_config.test_mpl" in results + else: + assert not json_summary.exists() + + html_summary = path / "fig_comparison.html" + if "html" in expected: + with open(html_summary) as fp: + raw = fp.read() + assert "bootstrap" in raw + assert "test_config.test_mpl" in raw + else: + assert not html_summary.exists() + + basic_html_summary = path / "fig_comparison_basic.html" + if "basic-html" in expected: + with open(basic_html_summary) as fp: + raw = fp.read() + assert "bootstrap" not in raw + assert "test_config.test_mpl" in raw + else: + assert not basic_html_summary.exists() diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py new file mode 100644 index 0000000..58c457b --- /dev/null +++ b/tests/test_hash_library.py @@ -0,0 +1,50 @@ +import json + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + ("bad", None, None, False), + ("good", None, None, True), + ("bad", "good", None, True), + ("bad", "bad", "good", False), # Note: CLI overrides kwarg + ("bad", "good", "bad", True), + ], +) +def test_config(pytester, ini, cli, kwarg, success_expected): + path = pytester_path(pytester) + hash_libraries = { + "good": path / "good_hash_library.json", + "bad": path / "bad_hash_library.json", + } + ini = f"mpl-hash-library = {hash_libraries[ini]}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"hash_library=r'{hash_libraries[kwarg]}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 3, 2]) + return fig + """ + ) + pytester.runpytest(f"--mpl-generate-hash-library={hash_libraries['good']}") + with open(hash_libraries["bad"], "w") as fp: + json.dump({"test_config.test_mpl": "bad-value"}, fp) + cli = f"--mpl-hash-library={hash_libraries[cli]}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_results_always.py b/tests/test_results_always.py new file mode 100644 index 0000000..ff2cc84 --- /dev/null +++ b/tests/test_results_always.py @@ -0,0 +1,43 @@ +from pathlib import Path + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, enabled_expected", + [ + (None, None, False), + (True, None, True), + (False, None, False), + (False, True, True), + (True, True, True), + ], +) +def test_config(pytester, ini, cli, enabled_expected): + path = pytester_path(pytester) + ini = f"mpl-results-always = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {Path(__file__).parent / "baseline" / "2.0.x"} + mpl-results-path = {path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_base_style(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-results-always" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(passed=1) + assert (path / "test_config.test_base_style" / "result.png").exists() == enabled_expected diff --git a/tests/test_results_path.py b/tests/test_results_path.py new file mode 100644 index 0000000..a727c2a --- /dev/null +++ b/tests/test_results_path.py @@ -0,0 +1,35 @@ +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("dir1", None, "dir1"), + ("dir1", "dir2", "dir2"), + (None, "dir2", "dir2"), + ], +) +def test_config(pytester, ini, cli, expected): + ini = f"mpl-results-path = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-results-path={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + assert (pytester_path(pytester) / expected / "test_config.test_mpl" / "result.png").exists() diff --git a/tests/test_use_full_test_name.py b/tests/test_use_full_test_name.py new file mode 100644 index 0000000..56aaca5 --- /dev/null +++ b/tests/test_use_full_test_name.py @@ -0,0 +1,54 @@ +import shutil +from pathlib import Path + +import pytest +from helpers import pytester_path + +FULL_TEST_NAME = "test_config.TestClass.test_mpl" +SHORT_TEST_NAME = "test_mpl" + + +@pytest.mark.parametrize( + "ini, cli, expected_baseline_name, success_expected", + [ + (None, None, SHORT_TEST_NAME, True), + (False, None, SHORT_TEST_NAME, True), + (True, None, FULL_TEST_NAME, True), + (False, True, FULL_TEST_NAME, True), + (None, True, FULL_TEST_NAME, True), + (True, True, "bad_name", False), + ], +) +def test_config(pytester, ini, cli, expected_baseline_name, success_expected): + path = pytester_path(pytester) + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + path / f"{expected_baseline_name}.png", + ) + ini = f"mpl-use-full-test-name = {ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + class TestClass: + @pytest.mark.mpl_image_compare + def test_mpl(self): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-use-full-test-name" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1)