Implement a persist marker. (#28)

Author: tobiasraabe

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     -   id: blacken-docs
         additional_dependencies: [black]
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.8.4
     hooks:
     -   id: flake8
         additional_dependencies: [

docs/changes.rst

@@ -13,6 +13,7 @@ all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>
 - :gh:`26` makes commands return the correct exit codes.
 - :gh:`27` implements the ``pytask_collect_task_teardown`` hook specification to perform
   checks after a task is collected.
+- :gh:`28` implements the ``@pytask.mark.persist`` decorator.
 
 
 0.0.6 - 2020-09-12

docs/explanations/why_another_build_system.rst

@@ -66,6 +66,24 @@ Cons
 - Bus factor of 1.
 
 
+`Luigi <https://github.com/spotify/luigi>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A build system written by Spotify.
+
+Derivatives:
+
+- `sciluigi <https://github.com/pharmbio/sciluigi>`_
+
+
+`scipipe <https://github.com/scipipe/scipipe>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cons
+
+- written in Go.
+
+
 `Scons <https://github.com/SCons/scons>`_
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/tutorials/how_to_make_tasks_persist.rst

@@ -0,0 +1,90 @@
+How to make tasks persist
+=========================
+
+You are able to create persisting tasks with pytask. It means that if all dependencies
+and products exist, the task will not be executed even though a dependency, the task's
+source file or a product has changed. Instead, the state of the dependencies, the source
+file and the products is updated in the database such that the next execution will skip
+the task successfully.
+
+When is this useful?
+--------------------
+
+1. You ran a formatter like Black against your project and there are some expensive
+   tasks which should not be executed.
+
+2. You want to integrate a task which you have already run elsewhere. Place the
+   dependencies and products and the task definition in the correct place and make the
+   task persist.
+
+
+.. caution::
+
+    This feature can corrupt the integrity of your project. At best, document why you
+    have applied the decorator as a courtesy to yourself and your contributors.
+
+
+How to do it?
+-------------
+
+To create a persisting task, apply the correct decorator and, et voilà, it is done.
+
+Let us take the second scenario as an example. First, we define the tasks, the
+dependency and the product and save everything in the same folder.
+
+.. code-block:: python
+
+    # Content of task_file.py
+
+    import pytask
+
+
+    @pytask.mark.persist
+    @pytask.mark.depends_on("input.md")
+    @pytask.mark.produces("output.md")
+    def task_make_input_bold(depends_on, produces):
+        produces.write_text("**" + depends_on.read_text() + "**")
+
+
+.. code-block::
+
+    # Content of input.md. Do not copy this line.
+
+    Here is the text.
+
+
+.. code-block::
+
+    # Content of output.md. Do not copy this line.
+
+    **Here is the text.**
+
+
+If you run pytask in this folder, you get the following output.
+
+.. code-block:: console
+
+    $ pytask demo
+    ========================= Start pytask session =========================
+    Platform: win32 -- Python 3.8.5, pytask 0.0.6, pluggy 0.13.1
+    Root: xxx/demo
+    Collected 1 task(s).
+
+    p
+    ====================== 1 succeeded in 1 second(s) ======================
+
+The green p signals that the task persisted. Another execution will show the following.
+
+.. code-block:: console
+
+    $ pytask demo
+    ========================= Start pytask session =========================
+    Platform: win32 -- Python 3.8.5, pytask 0.0.6, pluggy 0.13.1
+    Root: xxx/demo
+    Collected 1 task(s).
+
+    s
+    ====================== 1 succeeded in 1 second(s) ======================
+
+Now, the task is skipped successfully because nothing has changed compared to the
+previous run.

docs/tutorials/index.rst

@@ -16,3 +16,4 @@ organize and start your own project.
    how_to_configure_pytask
    how_to_select_tasks
    how_to_clean
+   how_to_make_tasks_persist

src/_pytask/cli.py

@@ -46,6 +46,7 @@ def pytask_add_hooks(pm):
     from _pytask import logging
     from _pytask import mark
     from _pytask import parametrize
+    from _pytask import persist
     from _pytask import resolve_dependencies
     from _pytask import skipping
 
@@ -59,6 +60,7 @@ def pytask_add_hooks(pm):
     pm.register(logging)
     pm.register(mark)
     pm.register(parametrize)
+    pm.register(persist)
     pm.register(resolve_dependencies)
     pm.register(skipping)
 

src/_pytask/outcomes.py

@@ -16,3 +16,7 @@ class SkippedDependencyNotFound(PytaskOutcome):
 
 class SkippedUnchanged(PytaskOutcome):
     """Outcome if task has run before and is unchanged."""
+
+
+class Persisted(PytaskOutcome):
+    """Outcome if task should persist."""

src/_pytask/persist.py

@@ -0,0 +1,65 @@
+"""Implement the ability for tasks to persist."""
+import click
+from _pytask.config import hookimpl
+from _pytask.dag import node_and_neighbors
+from _pytask.enums import ColorCode
+from _pytask.exceptions import NodeNotFoundError
+from _pytask.mark import get_specific_markers_from_task
+from _pytask.outcomes import Persisted
+
+
+@hookimpl
+def pytask_parse_config(config):
+    """Add the marker to the configuration."""
+    config["markers"]["persist"] = (
+        "Prevent execution of a task if all products exist and even if something has "
+        "changed (dependencies, source file, products). This decorator might be useful "
+        "for expensive tasks where only the formatting of the has changed. The state "
+        "of the files which have changed will also be remembered and another run will "
+        "skip the task with success."
+    )
+
+
+@hookimpl
+def pytask_execute_task_setup(session, task):
+    """Exit persisting tasks early.
+
+    The decorator needs to be set and all nodes need to exist.
+
+    """
+    if get_specific_markers_from_task(task, "persist"):
+        try:
+            for name in node_and_neighbors(session.dag, task.name):
+                node = (
+                    session.dag.nodes[name].get("task")
+                    or session.dag.nodes[name]["node"]
+                )
+                node.state()
+        except NodeNotFoundError:
+            all_nodes_exist = False
+        else:
+            all_nodes_exist = True
+
+        if all_nodes_exist:
+            raise Persisted
+
+
+@hookimpl
+def pytask_execute_task_process_report(report):
+    """Set task status to success.
+
+    Do not return ``True`` so that states will be updated in database.
+
+    """
+    if report.exc_info and isinstance(report.exc_info[1], Persisted):
+        report.success = True
+
+
+@hookimpl
+def pytask_execute_task_log_end(report):
+    """Log a persisting task with a green p."""
+    if report.success:
+        if report.exc_info:
+            if isinstance(report.exc_info[1], Persisted):
+                click.secho("p", fg=ColorCode.SUCCESS.value, nl=False)
+                return True

tests/test_persist.py

@@ -0,0 +1,125 @@
+import textwrap
+
+import pytest
+from _pytask.database import create_database
+from _pytask.database import State
+from _pytask.outcomes import Persisted
+from _pytask.outcomes import SkippedUnchanged
+from _pytask.persist import pytask_execute_task_process_report
+from pony import orm
+from pytask import main
+
+
+class DummyClass:
+    pass
+
+
+@pytest.mark.end_to_end
+def test_persist_marker_is_set(tmp_path):
+    session = main({"paths": tmp_path})
+    assert "persist" in session.config["markers"]
+
+
+@pytest.mark.end_to_end
+def test_multiple_runs_with_persist(tmp_path):
+    """Perform multiple consecutive runs and check intermediate outcomes with persist.
+
+    1. The product is missing which should result in a normal execution of the task.
+    2. Change the product, check that run is successful and state in database has
+       changed.
+    3. Run the task another time. Now, the task is skipped successfully.
+
+    """
+    source = """
+    import pytask
+
+    @pytask.mark.persist
+    @pytask.mark.depends_on("in.txt")
+    @pytask.mark.produces("out.txt")
+    def task_dummy(depends_on, produces):
+        produces.write_text(depends_on.read_text())
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source))
+    tmp_path.joinpath("in.txt").write_text("I'm not the reason you care.")
+
+    session = main({"paths": tmp_path})
+
+    assert session.exit_code == 0
+    assert len(session.execution_reports) == 1
+    assert session.execution_reports[0].success
+    assert session.execution_reports[0].exc_info is None
+    assert tmp_path.joinpath("out.txt").exists()
+
+    tmp_path.joinpath("out.txt").write_text("Never again in despair.")
+
+    session = main({"paths": tmp_path})
+
+    assert session.exit_code == 0
+    assert len(session.execution_reports) == 1
+    assert session.execution_reports[0].success
+    assert isinstance(session.execution_reports[0].exc_info[1], Persisted)
+
+    orm.db_session.__enter__()
+
+    create_database(
+        "sqlite", tmp_path.joinpath(".pytask.sqlite3").as_posix(), True, False
+    )
+    task_id = tmp_path.joinpath("task_dummy.py").as_posix() + "::task_dummy"
+    node_id = tmp_path.joinpath("out.txt").as_posix()
+
+    state = State[task_id, node_id].state
+    assert float(state) == tmp_path.joinpath("out.txt").stat().st_mtime
+
+    orm.db_session.__exit__()
+
+    session = main({"paths": tmp_path})
+
+    assert session.exit_code == 0
+    assert len(session.execution_reports) == 1
+    assert session.execution_reports[0].success
+    assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged)
+
+
+@pytest.mark.end_to_end
+def test_migrating_a_whole_task_with_persist(tmp_path):
+    source = """
+    import pytask
+
+    @pytask.mark.persist
+    @pytask.mark.depends_on("in.txt")
+    @pytask.mark.produces("out.txt")
+    def task_dummy(depends_on, produces):
+        produces.write_text(depends_on.read_text())
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source))
+    for name in ["in.txt", "out.txt"]:
+        tmp_path.joinpath(name).write_text(
+            "They say oh my god I see the way you shine."
+        )
+
+    session = main({"paths": tmp_path})
+
+    assert session.exit_code == 0
+    assert len(session.execution_reports) == 1
+    assert session.execution_reports[0].success
+    assert isinstance(session.execution_reports[0].exc_info[1], Persisted)
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "exc_info, expected",
+    [
+        (None, None),
+        ((None, None, None), None),
+        ((None, Persisted(), None), True),
+    ],
+)
+def test_pytask_execute_task_process_report(exc_info, expected):
+    report = DummyClass()
+    report.exc_info = exc_info
+    result = pytask_execute_task_process_report(report)
+
+    if expected:
+        assert report.success
+    else:
+        assert result is None