From 82d4cac7a4fc5c98f9e4c73df99525e5d77e4822 Mon Sep 17 00:00:00 2001 From: Thomas S Date: Tue, 9 Jul 2024 16:36:31 +0200 Subject: [PATCH 1/4] Structure the test directory --- tests/{__init__.py => conftest.py} | 0 tests/functional/.gitkeep | 0 tests/integration/.gitkeep | 0 tests/{test_mander_dsl.py => unit/test_infomander.py} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tests/{__init__.py => conftest.py} (100%) create mode 100644 tests/functional/.gitkeep create mode 100644 tests/integration/.gitkeep rename tests/{test_mander_dsl.py => unit/test_infomander.py} (100%) diff --git a/tests/__init__.py b/tests/conftest.py similarity index 100% rename from tests/__init__.py rename to tests/conftest.py diff --git a/tests/functional/.gitkeep b/tests/functional/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/.gitkeep b/tests/integration/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_mander_dsl.py b/tests/unit/test_infomander.py similarity index 100% rename from tests/test_mander_dsl.py rename to tests/unit/test_infomander.py From bd6b32cbec38fc948a176e2cf4d7e7a1b8a31f67 Mon Sep 17 00:00:00 2001 From: Thomas S Date: Wed, 10 Jul 2024 16:56:45 +0200 Subject: [PATCH 2/4] Add InfoMander tests --- tests/conftest.py | 34 ++++++ tests/unit/test_infomander.py | 188 ++++++++++++++++++++++++++++++++-- 2 files changed, 215 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e69de29b..7f6f07cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -0,0 +1,34 @@ +from datetime import UTC, datetime + +import pytest +from mandr import InfoMander + + +@pytest.fixture +def mock_now(): + return datetime.now(tz=UTC) + + +@pytest.fixture +def mock_nowstr(mock_now): + return mock_now.isoformat() + + +@pytest.fixture +def mock_mandr(monkeypatch, mock_now, tmp_path): + class MockCache(dict): + def __init__(self, *args, **kwargs): + super().__init__() + + def iterkeys(self, *args, **kwargs): + yield from self.keys() + + class MockDatetime: + @staticmethod + def now(*args, **kwargs): + return mock_now + + monkeypatch.setattr("mandr.infomander.Cache", MockCache) + monkeypatch.setattr("mandr.infomander.datetime", MockDatetime) + + return InfoMander("root", root=tmp_path) diff --git a/tests/unit/test_infomander.py b/tests/unit/test_infomander.py index 11d0eb71..20eeacfb 100644 --- a/tests/unit/test_infomander.py +++ b/tests/unit/test_infomander.py @@ -1,9 +1,183 @@ -""" -The mander comes with a special DSL to fetch the right elements from the cache. -This is tested here. -""" +import joblib +import pytest +from mandr import InfoMander -def test_the_obvious(): - """Temporary test to ensure that CI runs it.""" - assert 2 + 2 == 4 +class TestInfoMander: + def test_check_key(self): + for key in InfoMander.RESERVED_KEYS: + with pytest.raises(ValueError): + InfoMander._check_key(key) + + def test_add_info_overwrite_true(self, mock_nowstr, mock_mandr): + mock_mandr.add_info("key1", "value1", overwrite=True) + mock_mandr.add_info("key2", "value1", overwrite=True) + mock_mandr.add_info("key2", "value2", overwrite=True) + + assert mock_mandr.cache == { + "artifacts": {}, + "templates": {}, + "views": {}, + "logs": {}, + "key1": "value1", + "key2": "value2", + "updated_at": mock_nowstr, + } + + def test_add_info_overwrite_false(self, mock_nowstr, mock_mandr): + mock_mandr.add_info("key1", "value1", overwrite=False) + mock_mandr.add_info("key2", ["value1"], overwrite=False) + mock_mandr.add_info("key2", ["value2"], overwrite=False) + mock_mandr.add_info("key3", "value1", overwrite=False) + mock_mandr.add_info("key3", "value2", overwrite=False) + + assert mock_mandr.cache == { + "artifacts": {}, + "templates": {}, + "views": {}, + "logs": {}, + "key1": ["value1"], + "key2": ["value2", "value1"], + "key3": ["value2", "value1"], + "updated_at": mock_nowstr, + } + + def test_add_to_key(self, mock_nowstr, mock_mandr): + mock_mandr._add_to_key("artifacts", "key", "value") + + assert mock_mandr.cache == { + "artifacts": {"key": "value"}, + "templates": {}, + "views": {}, + "logs": {}, + "updated_at": mock_nowstr, + } + + def test_add_artifact(self, mock_now, mock_nowstr, mock_mandr, tmp_path): + mock_mandr.add_artifact("key", "value", datetime=mock_now) + + assert joblib.load(tmp_path / "root" / ".artifacts" / "key.joblib") == "value" + assert mock_mandr.cache == { + "artifacts": {"key": {"datetime": mock_now}}, + "templates": {}, + "views": {}, + "logs": {}, + "updated_at": mock_nowstr, + } + + def test_add_view(self, mock_nowstr, mock_mandr): + mock_mandr.add_view("key", "hydrated") + + assert mock_mandr.cache == { + "artifacts": {}, + "templates": {}, + "views": {"key": "hydrated"}, + "logs": {}, + "updated_at": mock_nowstr, + } + + def test_add_template(self, mock_nowstr, mock_mandr): + mock_mandr.add_template("key", "{{template}}") + + assert mock_mandr.cache == { + "artifacts": {}, + "templates": {"key": "{{template}}"}, + "views": {}, + "logs": {}, + "updated_at": mock_nowstr, + } + + def test_add_template_exception(self, mock_mandr): + mock_mandr.cache["views"] = {"key": "hydrated"} + + with pytest.raises(ValueError): + mock_mandr.add_template("key", "{{template}}") + + def test_render_templates(self, mock_nowstr, mock_mandr): + class MockTemplate: + def render(self, *args, **kwargs): + return "hydrated" + + template = MockTemplate() + mock_mandr.cache["templates"] = {"key": template} + mock_mandr.render_templates() + + assert mock_mandr.cache == { + "artifacts": {}, + "templates": {"key": template}, + "views": {"key": "hydrated"}, + "logs": {}, + "updated_at": mock_nowstr, + } + + def test_add_logs(self, mock_nowstr, mock_mandr): + mock_mandr.add_logs("key", "value") + + assert mock_mandr.cache == { + "artifacts": {}, + "templates": {}, + "views": {}, + "logs": {"key": "value"}, + "updated_at": mock_nowstr, + } + + def test_fetch(self, mock_mandr): + assert mock_mandr.fetch() == { + "artifacts": {}, + "templates": {}, + "views": {}, + "logs": {}, + } + + def test_getitem(self, mock_mandr): + assert mock_mandr["artifacts"] == {} + + def test_dsl_path_exists(self, mock_mandr, tmp_path): + (tmp_path / "root2").mkdir(parents=True) + (tmp_path / "root3").mkdir(parents=True) + + assert mock_mandr.dsl_path_exists("root2") is None + assert mock_mandr.dsl_path_exists("root3") is None + + with pytest.raises(AssertionError): + mock_mandr.dsl_path_exists("root4") + + def test_child(self, mock_mandr, tmp_path): + tmp_path /= "root" + + (tmp_path).mkdir(parents=True) + (tmp_path / "subroot1").mkdir(parents=True) + + assert mock_mandr.get_child("subroot1") == InfoMander("subroot1", root=tmp_path) + assert mock_mandr.get_child("subroot2") == InfoMander("subroot2", root=tmp_path) + + def test_children(self, mock_mandr, tmp_path): + tmp_path /= "root" + + (tmp_path).mkdir(parents=True) + (tmp_path / ".artifacts").mkdir(parents=True) + (tmp_path / ".stats").mkdir(parents=True) + + assert mock_mandr.children() == [] + + (tmp_path / "subroot1").mkdir(parents=True) + (tmp_path / "subroot2").mkdir(parents=True) + + assert mock_mandr.children() == [ + InfoMander("subroot1", root=tmp_path), + InfoMander("subroot2", root=tmp_path), + ] + + def test_repr(self, mock_mandr, tmp_path): + assert repr(mock_mandr) == f"InfoMander({tmp_path / 'root'})" + + def test_eq(self, tmp_path): + IM = InfoMander + + assert IM("root", root=tmp_path) == IM("root", root=tmp_path) + assert IM("root/subroot", root=tmp_path) == IM("root/subroot", root=tmp_path) + assert IM("root1", root=tmp_path) != IM("root2", root=tmp_path) + assert IM("root1/subroot", root=tmp_path) != IM("root2/subroot", root=tmp_path) + assert IM("root1/subroot1", root=tmp_path) != IM( + "root1/subroot2", root=tmp_path + ) From 669dde7837accca49388c8f823ccea165c2c7dbf Mon Sep 17 00:00:00 2001 From: Thomas S Date: Wed, 10 Jul 2024 17:06:24 +0200 Subject: [PATCH 3/4] Fix InfoMander to pass tests --- src/mandr/infomander.py | 122 ++++++++++++++++++++-------------- tests/unit/test_infomander.py | 4 +- 2 files changed, 75 insertions(+), 51 deletions(-) diff --git a/src/mandr/infomander.py b/src/mandr/infomander.py index 2d3cbc21..ce658c0a 100644 --- a/src/mandr/infomander.py +++ b/src/mandr/infomander.py @@ -1,54 +1,68 @@ """Contains the code for the main InfoMander class.""" +from datetime import UTC, datetime from pathlib import Path -from time import time from diskcache import Cache from joblib import dump -from rich.console import Console from .templates import TemplateRenderer -console = Console() -LOGS_KEY = "logs" -VIEWS_KEY = "views" -TEMPLATES_KEY = "templates" -ARTIFACTS_KEY = "artifacts" - -STATS_FOLDER = ".stats" -ARTIFACTS_FOLDER = ".artifacts" -LOGS_FOLDER = ".logs" - class InfoMander: """Represents a dictionary, on disk, with a path-like structure.""" - def __init__(self, path): + ARTIFACTS_KEY = "artifacts" + LOGS_KEY = "logs" + TEMPLATES_KEY = "templates" + VIEWS_KEY = "views" + + RESERVED_KEYS = { + ARTIFACTS_KEY, + LOGS_KEY, + TEMPLATES_KEY, + VIEWS_KEY, + } + + STATS_FOLDER = ".stats" + ARTIFACTS_FOLDER = ".artifacts" + + RESERVED_FOLDERS = { + STATS_FOLDER, + ARTIFACTS_FOLDER, + } + + def __init__( + self, path: str, /, *, root: Path = (Path(__file__).parent / ".datamander") + ): + if Path(path).is_absolute(): + raise ValueError("Cant use absolute path") + # Set local disk paths - self.path = path - self.project_path = Path(".datamander/" + path) - self.cache = Cache(self.project_path / STATS_FOLDER) + self.root_path = root + self.project_path = self.root_path / path + self.cache = Cache(self.project_path / InfoMander.STATS_FOLDER) - # For practical reasons the logs and artifacts are stored on disk, not sqlite + # For practical reasons the artifacts are stored on disk, not sqlite # We could certainly revisit this later though - self.artifact_path = self.project_path / ARTIFACTS_FOLDER - self.log_path = self.project_path / LOGS_FOLDER + self.artifact_path = self.project_path / InfoMander.ARTIFACTS_FOLDER # Initialize the internal cache with empty values if need be - for key in [ARTIFACTS_KEY, TEMPLATES_KEY, VIEWS_KEY, LOGS_KEY]: + for key in InfoMander.RESERVED_KEYS: if key not in self.cache: self.cache[key] = {} # This will be used for rendering templates into views self.renderer = TemplateRenderer(self) - def _check_key(self, key): - if key in [ARTIFACTS_KEY, TEMPLATES_KEY, VIEWS_KEY, LOGS_KEY]: + @staticmethod + def _check_key(key): + if key in InfoMander.RESERVED_KEYS: raise ValueError( - f"Cannot overwrite {key} key. This is reserved for internal use." + f"Cannot overwrite '{key}' key. This is reserved for internal use." ) - def add_info(self, key, value, method="overwrite"): + def add_info(self, key, value, overwrite=True): """ Add information to the cache. Can be appended or overwritten. @@ -58,18 +72,19 @@ def add_info(self, key, value, method="overwrite"): The key to add the information to. value : any The value to add. - method : str - The method to use. Can be 'overwrite' or 'append'. + overwrite : bool, default=True + Overwrite value, otherwise append. """ - if method == "overwrite": + if overwrite: self.cache[key] = value - if method == "append": + else: if not isinstance(value, list): value = [value] if key in self.cache: value = value + self.cache[key] self.cache[key] = value - self.cache["updated_at"] = int(time()) + + self.cache["updated_at"] = datetime.now(tz=UTC).isoformat() def _add_to_key(self, top_key, key, value): """ @@ -113,7 +128,7 @@ def add_artifact(self, key, obj, **metadata): if not file_location.parent.exists(): file_location.parent.mkdir(parents=True) dump(obj, file_location) - self._add_to_key(ARTIFACTS_KEY, key, {"obj": obj, **metadata}) + self._add_to_key(InfoMander.ARTIFACTS_KEY, key, metadata) def add_view(self, key, html): """ @@ -130,7 +145,7 @@ def add_view(self, key, html): The HTML to store. """ self._check_key(key) - self._add_to_key(VIEWS_KEY, key, html) + self._add_to_key(InfoMander.VIEWS_KEY, key, html) def add_template(self, key, template): """ @@ -147,11 +162,13 @@ def add_template(self, key, template): The template to store. """ self._check_key(key) - if key in self.cache[VIEWS_KEY]: + + if key in self.cache[InfoMander.VIEWS_KEY]: raise ValueError( - f"Cannot add template {key} because that name is already present." + f"Unable to add template '{key}': view rendered with this key name." ) - self._add_to_key("_templates", key, template) + + self._add_to_key(InfoMander.TEMPLATES_KEY, key, template) def render_templates(self): """ @@ -164,14 +181,14 @@ def render_templates(self): template : Template The template to store. """ - for name, template in self.cache["_templates"].items(): + for name, template in self.cache[InfoMander.TEMPLATES_KEY].items(): self.add_view(name, template.render(self)) def add_logs(self, key, logs): """ Add logs to the cache. - Logs are stored separately in the cache and can be viewed in the web ui. + Logs are stored in the cache and can be viewed in the web ui. Parameters ---------- @@ -180,7 +197,7 @@ def add_logs(self, key, logs): logs : list The logs to store. """ - self._add_to_key(LOGS_KEY, key, logs) + self._add_to_key(InfoMander.LOGS_KEY, key, logs) def fetch(self): """Return all information from the cache.""" @@ -190,8 +207,8 @@ def __getitem__(self, key): """Return a specific item from the cache.""" return self.cache[key] - @classmethod - def get_property(cls, mander, dsl_str): + @staticmethod + def get_property(mander, dsl_str): """ Get a property from a mander using the DSL syntax. @@ -232,29 +249,28 @@ def dsl_path_exists(self, path): path : str The path to check for. """ - actual_path = ".datamander" / path - assert Path(actual_path).exists() + assert (self.root_path / path).exists() - def get_child(self, *path): + def get_child(self, *pathsegments: str): """ Get a child mander from the current mander. Parameters ---------- - path : str - The path to the child mander + *pathsegments : str + The path to the child mander. """ - new_path = self.project_path - for p in path: - new_path = new_path / p - return InfoMander(str(new_path)) + return InfoMander(Path(*pathsegments), root=self.project_path) def children(self): """Return all children of the mander.""" return [ - InfoMander("/".join(p.parts[1:])) + InfoMander( + p.relative_to(self.project_path), + root=self.project_path, + ) for p in self.project_path.iterdir() - if p.is_dir() and not p.name.startswith(".") + if p.is_dir() and (p.stem not in InfoMander.RESERVED_FOLDERS) ] def get(self, dsl_str): @@ -280,3 +296,9 @@ def get(self, dsl_str): def __repr__(self): """Return a string representation of the mander.""" return f"InfoMander({self.project_path})" + + def __eq__(self, other): + """Return True if both instances have the same project path.""" + if isinstance(other, InfoMander): + return self.project_path == other.project_path + return False diff --git a/tests/unit/test_infomander.py b/tests/unit/test_infomander.py index 20eeacfb..cdd1ca36 100644 --- a/tests/unit/test_infomander.py +++ b/tests/unit/test_infomander.py @@ -1,3 +1,5 @@ +from operator import attrgetter + import joblib import pytest from mandr import InfoMander @@ -163,7 +165,7 @@ def test_children(self, mock_mandr, tmp_path): (tmp_path / "subroot1").mkdir(parents=True) (tmp_path / "subroot2").mkdir(parents=True) - assert mock_mandr.children() == [ + assert sorted(mock_mandr.children(), key=attrgetter("project_path")) == [ InfoMander("subroot1", root=tmp_path), InfoMander("subroot2", root=tmp_path), ] From 8955919917d2b11d48383a1259cd0bbd9e4f93f5 Mon Sep 17 00:00:00 2001 From: Thomas S Date: Wed, 10 Jul 2024 17:07:13 +0200 Subject: [PATCH 4/4] Fix pytest default parameters --- .github/workflows/lint-and-test.yml | 5 ++--- pyproject.toml | 12 ++++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 4b2240e2..dae98785 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -21,11 +21,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pre-commit - pip install -e . -r requirements.txt -r requirements-test.txt + python -m pip install -e . -r requirements.txt -r requirements-test.txt - name: Lint with pre-commit run: | pre-commit run --all-files - name: Test with pytest run: | - pytest --cov + python -m pytest tests diff --git a/pyproject.toml b/pyproject.toml index 4fa05565..7098c43c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,10 @@ dependencies = [ [project.optional-dependencies] test = [ + "pre-commit", "pytest", "pytest-cov", - "pre-commit", + "pytest-randomly", "ruff" ] @@ -35,7 +36,14 @@ packages = ["src/mandr"] upgrade = true [tool.pytest.ini_options] -addopts = ["--import-mode=importlib"] +addopts = [ + "--cov=src/", + "--cov=tests/", + "--cov-branch", + "--import-mode=importlib", + "--no-header", + "--verbosity=2", +] [tool.ruff] src = ["mandr"]