diff --git a/setup.cfg b/setup.cfg index 7f9a3e3..f8026f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ filterwarnings = error ignore:distutils Version classes are deprecated ignore:the imp module is deprecated in favour of importlib + ignore:The NumPy module was reloaded [flake8] max-line-length = 100 diff --git a/tests/test_baseline_path.py b/tests/test_baseline_path.py new file mode 100644 index 0000000..1517ac1 --- /dev/null +++ b/tests/test_baseline_path.py @@ -0,0 +1,48 @@ +import shutil +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected_baseline_path, success_expected", + [ + ("dir1", None, None, "dir1", True), + ("dir1", "dir2", None, "dir2", True), + ("dir1", "dir2", "dir3", "dir3", True), + ("dir1", "dir2", "dir3", "dir2", False), + (None, None, "dir3", "dir3", True), + ], +) +def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected): + (pytester.path / expected_baseline_path).mkdir() + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + pytester.path / expected_baseline_path / "test_mpl.png", + ) + ini = f"mpl-baseline-path = {pytester.path / ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + {ini} + """ + ) + kwarg = f"baseline_dir='{pytester.path / kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-baseline-path={pytester.path / cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_default_backend.py b/tests/test_default_backend.py new file mode 100644 index 0000000..b84624a --- /dev/null +++ b/tests/test_default_backend.py @@ -0,0 +1,35 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("backend1", None, None, "backend1"), + ("backend1", "backend2", None, "backend2"), + ("backend1", "backend2", "backend3", "backend3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = f"mpl-default-backend = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"backend='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-backend={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*ModuleNotFoundError: No module named 'matplotlib.backends.backend_{expected}'*"]) diff --git a/tests/test_default_style.py b/tests/test_default_style.py new file mode 100644 index 0000000..1b0c1e6 --- /dev/null +++ b/tests/test_default_style.py @@ -0,0 +1,37 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("sty1", None, None, "sty1"), + ("sty1", "sty2", None, "sty2"), + ("sty1", "sty2", "sty3", "sty3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = "mpl-default-style = " + ini if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-baseline-path = {pytester.path} + {ini} + """ + ) + kwarg = f"style='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + + cli = "--mpl-default-style=" + cli if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*OSError: '{expected}' is not a valid package style*"]) diff --git a/tests/test_default_tolerance.py b/tests/test_default_tolerance.py new file mode 100644 index 0000000..c12832c --- /dev/null +++ b/tests/test_default_tolerance.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import pytest +from PIL import Image, ImageDraw + +TEST_NAME = "test_base_style" + + +@pytest.fixture(scope="module") +def baseline_image(tmpdir_factory): + path = Path(__file__).parent / "baseline" / "2.0.x" / f"{TEST_NAME}.png" + image = Image.open(path) + draw = ImageDraw.Draw(image) + draw.rectangle(((0, 0), (100, 100)), fill="red") + output = Path(tmpdir_factory.mktemp("data").join(f"{TEST_NAME}.png")) + print(output) + image.save(output) + return output + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + (40, None, None, True), + (30, 40, None, True), + (30, 30, 40, True), + (30, 40, 30, False), + (40, 30, 30, False), + ], +) +def test_config(pytester, baseline_image, ini, cli, kwarg, success_expected): + ini = f"mpl-default-tolerance = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {baseline_image.parent} + {ini} + """ + ) + kwarg = f"tolerance={kwarg}" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def {TEST_NAME}(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-tolerance={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_generate_summary.py b/tests/test_generate_summary.py new file mode 100644 index 0000000..3632207 --- /dev/null +++ b/tests/test_generate_summary.py @@ -0,0 +1,63 @@ +import json + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("json", None, {"json"}), + ("json", "html", {"html"}), + ("basic-html", "json", {"json"}), + (None, "json,basic-html,html", {"json", "basic-html", "html"}), + ], +) +def test_config(pytester, ini, cli, expected): + ini = f"mpl-generate-summary = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-results-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-generate-summary={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + + json_summary = pytester.path / "results.json" + if "json" in expected: + with open(json_summary) as fp: + results = json.load(fp) + assert "test_config.test_mpl" in results + else: + assert not json_summary.exists() + + html_summary = pytester.path / "fig_comparison.html" + if "html" in expected: + with open(html_summary) as fp: + raw = fp.read() + assert "bootstrap" in raw + assert "test_config.test_mpl" in raw + else: + assert not html_summary.exists() + + basic_html_summary = pytester.path / "fig_comparison_basic.html" + if "basic-html" in expected: + with open(basic_html_summary) as fp: + raw = fp.read() + assert "bootstrap" not in raw + assert "test_config.test_mpl" in raw + else: + assert not basic_html_summary.exists() diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py new file mode 100644 index 0000000..29a01c6 --- /dev/null +++ b/tests/test_hash_library.py @@ -0,0 +1,52 @@ +import json + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + ("bad", None, None, False), + ("good", None, None, True), + ("bad", "good", None, True), + ("bad", "bad", "good", False), # Note: CLI overrides kwarg + ("bad", "good", "bad", True), + ], +) +def test_config(pytester, ini, cli, kwarg, success_expected): + hash_libraries = { + "good": (pytester.path / "good_hash_library.json", + "b1e03274b2df4130e0894afd6c0faa76805e1851eec32f38e86d2423de0d9186"), + "bad": (pytester.path / "bad_hash_library.json", "bad-value"), + } + for library_path, hash_val in hash_libraries.values(): + with open(library_path, "w") as fp: + json.dump({f"test_config.test_mpl": hash_val}, fp) + + ini = f"mpl-hash-library = {hash_libraries[ini][0]}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + + kwarg = f"hash_library='{hash_libraries[kwarg][0]}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 3, 2]) + return fig + """ + ) + + cli = f"--mpl-hash-library={hash_libraries[cli][0]}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_use_full_test_name.py b/tests/test_use_full_test_name.py new file mode 100644 index 0000000..f78bd48 --- /dev/null +++ b/tests/test_use_full_test_name.py @@ -0,0 +1,52 @@ +import shutil +from pathlib import Path + +import pytest + +FULL_TEST_NAME = "test_config.TestClass.test_mpl" +SHORT_TEST_NAME = "test_mpl" + + +@pytest.mark.parametrize( + "ini, cli, expected_baseline_name, success_expected", + [ + (None, None, SHORT_TEST_NAME, True), + (False, None, SHORT_TEST_NAME, True), + (True, None, FULL_TEST_NAME, True), + (False, True, FULL_TEST_NAME, True), + (None, True, FULL_TEST_NAME, True), + (True, True, "bad_name", False), + ], +) +def test_config(pytester, ini, cli, expected_baseline_name, success_expected): + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + pytester.path / f"{expected_baseline_name}.png", + ) + ini = f"mpl-use-full-test-name = {ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + class TestClass: + @pytest.mark.mpl_image_compare + def test_mpl(self): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-use-full-test-name" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1)