Test new config options

Adds tests for the new config options across ini, CLI and kwarg where relevant. Asserts that they work and have the expected order of precedence. Still need to add dedicated tests for all the other options.

setup.cfg

@@ -52,6 +52,7 @@ filterwarnings =
     error
     ignore:distutils Version classes are deprecated
     ignore:the imp module is deprecated in favour of importlib
+    ignore:The NumPy module was reloaded
 
 [flake8]
 max-line-length = 100

tests/test_baseline_path.py

@@ -0,0 +1,48 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "ini, cli, kwarg, expected_baseline_path, success_expected",
+    [
+        ("dir1", None, None, "dir1", True),
+        ("dir1", "dir2", None, "dir2", True),
+        ("dir1", "dir2", "dir3", "dir3", True),
+        ("dir1", "dir2", "dir3", "dir2", False),
+        (None, None, "dir3", "dir3", True),
+    ],
+)
+def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected):
+    (pytester.path / expected_baseline_path).mkdir()
+    shutil.copyfile(  # Test will only pass if baseline is at expected path
+        Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png",
+        pytester.path / expected_baseline_path / "test_mpl.png",
+    )
+    ini = f"mpl-baseline-path = {pytester.path / ini}" if ini is not None else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        mpl-default-style = fivethirtyeight
+        {ini}
+        """
+    )
+    kwarg = f"baseline_dir='{pytester.path / kwarg}'" if kwarg else ""
+    pytester.makepyfile(
+        f"""
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare({kwarg})
+        def test_mpl():
+            fig, ax = plt.subplots()
+            ax.plot([1, 2, 3])
+            return fig
+        """
+    )
+    cli = f"--mpl-baseline-path={pytester.path / cli}" if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    if success_expected:
+        result.assert_outcomes(passed=1)
+    else:
+        result.assert_outcomes(failed=1)

tests/test_default_backend.py

@@ -0,0 +1,35 @@
+import pytest
+
+
+@pytest.mark.parametrize(
+    "ini, cli, kwarg, expected",
+    [
+        ("backend1", None, None, "backend1"),
+        ("backend1", "backend2", None, "backend2"),
+        ("backend1", "backend2", "backend3", "backend3"),
+    ],
+)
+def test_config(pytester, ini, cli, kwarg, expected):
+    ini = f"mpl-default-backend = {ini}" if ini else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        {ini}
+        """
+    )
+    kwarg = f"backend='{kwarg}'" if kwarg else ""
+    pytester.makepyfile(
+        f"""
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare({kwarg})
+        def test_mpl():
+            fig, ax = plt.subplots()
+            ax.plot([1, 2, 3])
+            return fig
+        """
+    )
+    cli = f"--mpl-default-backend={cli}" if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    result.assert_outcomes(failed=1)
+    result.stdout.fnmatch_lines([f"*ModuleNotFoundError: No module named 'matplotlib.backends.backend_{expected}'*"])

tests/test_default_style.py

@@ -0,0 +1,37 @@
+import pytest
+
+
+@pytest.mark.parametrize(
+    "ini, cli, kwarg, expected",
+    [
+        ("sty1", None, None, "sty1"),
+        ("sty1", "sty2", None, "sty2"),
+        ("sty1", "sty2", "sty3", "sty3"),
+    ],
+)
+def test_config(pytester, ini, cli, kwarg, expected):
+    ini = "mpl-default-style = " + ini if ini else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        mpl-baseline-path = {pytester.path}
+        {ini}
+        """
+    )
+    kwarg = f"style='{kwarg}'" if kwarg else ""
+    pytester.makepyfile(
+        f"""
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare({kwarg})
+        def test_mpl():
+            fig, ax = plt.subplots()
+            ax.plot([1, 2, 3])
+            return fig
+        """
+    )
+
+    cli = "--mpl-default-style=" + cli if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    result.assert_outcomes(failed=1)
+    result.stdout.fnmatch_lines([f"*OSError: '{expected}' is not a valid package style*"])

tests/test_default_tolerance.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+
+import pytest
+from PIL import Image, ImageDraw
+
+TEST_NAME = "test_base_style"
+
+
+@pytest.fixture(scope="module")
+def baseline_image(tmpdir_factory):
+    path = Path(__file__).parent / "baseline" / "2.0.x" / f"{TEST_NAME}.png"
+    image = Image.open(path)
+    draw = ImageDraw.Draw(image)
+    draw.rectangle(((0, 0), (100, 100)), fill="red")
+    output = Path(tmpdir_factory.mktemp("data").join(f"{TEST_NAME}.png"))
+    print(output)
+    image.save(output)
+    return output
+
+
+@pytest.mark.parametrize(
+    "ini, cli, kwarg, success_expected",
+    [
+        (40, None, None, True),
+        (30, 40, None, True),
+        (30, 30, 40, True),
+        (30, 40, 30, False),
+        (40, 30, 30, False),
+    ],
+)
+def test_config(pytester, baseline_image, ini, cli, kwarg, success_expected):
+    ini = f"mpl-default-tolerance = {ini}" if ini else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        mpl-default-style = fivethirtyeight
+        mpl-baseline-path = {baseline_image.parent}
+        {ini}
+        """
+    )
+    kwarg = f"tolerance={kwarg}" if kwarg else ""
+    pytester.makepyfile(
+        f"""
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare({kwarg})
+        def {TEST_NAME}():
+            fig, ax = plt.subplots()
+            ax.plot([1, 2, 3])
+            return fig
+        """
+    )
+    cli = f"--mpl-default-tolerance={cli}" if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    if success_expected:
+        result.assert_outcomes(passed=1)
+    else:
+        result.assert_outcomes(failed=1)

tests/test_generate_summary.py

@@ -0,0 +1,63 @@
+import json
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "ini, cli, expected",
+    [
+        ("json", None, {"json"}),
+        ("json", "html", {"html"}),
+        ("basic-html", "json", {"json"}),
+        (None, "json,basic-html,html", {"json", "basic-html", "html"}),
+    ],
+)
+def test_config(pytester, ini, cli, expected):
+    ini = f"mpl-generate-summary = {ini}" if ini else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        mpl-results-path = {pytester.path}
+        {ini}
+        """
+    )
+    pytester.makepyfile(
+        """
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare
+        def test_mpl():
+            fig, ax = plt.subplots()
+            ax.plot([1, 2, 3])
+            return fig
+        """
+    )
+    cli = f"--mpl-generate-summary={cli}" if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    result.assert_outcomes(failed=1)
+
+    json_summary = pytester.path / "results.json"
+    if "json" in expected:
+        with open(json_summary) as fp:
+            results = json.load(fp)
+            assert "test_config.test_mpl" in results
+    else:
+        assert not json_summary.exists()
+
+    html_summary = pytester.path / "fig_comparison.html"
+    if "html" in expected:
+        with open(html_summary) as fp:
+            raw = fp.read()
+        assert "bootstrap" in raw
+        assert "test_config.test_mpl" in raw
+    else:
+        assert not html_summary.exists()
+
+    basic_html_summary = pytester.path / "fig_comparison_basic.html"
+    if "basic-html" in expected:
+        with open(basic_html_summary) as fp:
+            raw = fp.read()
+        assert "bootstrap" not in raw
+        assert "test_config.test_mpl" in raw
+    else:
+        assert not basic_html_summary.exists()

tests/test_hash_library.py

@@ -0,0 +1,52 @@
+import json
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "ini, cli, kwarg, success_expected",
+    [
+        ("bad", None, None, False),
+        ("good", None, None, True),
+        ("bad", "good", None, True),
+        ("bad", "bad", "good", False),  # Note: CLI overrides kwarg
+        ("bad", "good", "bad", True),
+    ],
+)
+def test_config(pytester, ini, cli, kwarg, success_expected):
+    hash_libraries = {
+        "good": (pytester.path / "good_hash_library.json",
+                 "b1e03274b2df4130e0894afd6c0faa76805e1851eec32f38e86d2423de0d9186"),
+        "bad": (pytester.path / "bad_hash_library.json", "bad-value"),
+    }
+    for library_path, hash_val in hash_libraries.values():
+        with open(library_path, "w") as fp:
+            json.dump({f"test_config.test_mpl": hash_val}, fp)
+
+    ini = f"mpl-hash-library = {hash_libraries[ini][0]}" if ini else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        {ini}
+        """
+    )
+
+    kwarg = f"hash_library='{hash_libraries[kwarg][0]}'" if kwarg else ""
+    pytester.makepyfile(
+        f"""
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare({kwarg})
+        def test_mpl():
+            fig, ax = plt.subplots()
+            ax.plot([1, 3, 2])
+            return fig
+        """
+    )
+
+    cli = f"--mpl-hash-library={hash_libraries[cli][0]}" if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    if success_expected:
+        result.assert_outcomes(passed=1)
+    else:
+        result.assert_outcomes(failed=1)

tests/test_use_full_test_name.py

@@ -0,0 +1,52 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+FULL_TEST_NAME = "test_config.TestClass.test_mpl"
+SHORT_TEST_NAME = "test_mpl"
+
+
+@pytest.mark.parametrize(
+    "ini, cli, expected_baseline_name, success_expected",
+    [
+        (None, None, SHORT_TEST_NAME, True),
+        (False, None, SHORT_TEST_NAME, True),
+        (True, None, FULL_TEST_NAME, True),
+        (False, True, FULL_TEST_NAME, True),
+        (None, True, FULL_TEST_NAME, True),
+        (True, True, "bad_name", False),
+    ],
+)
+def test_config(pytester, ini, cli, expected_baseline_name, success_expected):
+    shutil.copyfile(  # Test will only pass if baseline is at expected path
+        Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png",
+        pytester.path / f"{expected_baseline_name}.png",
+    )
+    ini = f"mpl-use-full-test-name = {ini}" if ini is not None else ""
+    pytester.makeini(
+        f"""
+        [pytest]
+        mpl-default-style = fivethirtyeight
+        mpl-baseline-path = {pytester.path}
+        {ini}
+        """
+    )
+    pytester.makepyfile(
+        """
+        import matplotlib.pyplot as plt
+        import pytest
+        class TestClass:
+            @pytest.mark.mpl_image_compare
+            def test_mpl(self):
+                fig, ax = plt.subplots()
+                ax.plot([1, 2, 3])
+                return fig
+        """
+    )
+    cli = "--mpl-use-full-test-name" if cli else ""
+    result = pytester.runpytest("--mpl", cli)
+    if success_expected:
+        result.assert_outcomes(passed=1)
+    else:
+        result.assert_outcomes(failed=1)