diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16b2035a..95cafe99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: rev: v2.0.0 hooks: - id: refurb + additional_dependencies: [typed-settings] - repo: https://github.com/executablebooks/mdformat rev: 0.7.17 hooks: diff --git a/docs/source/changes.md b/docs/source/changes.md index b29d9f55..39eb1b33 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -27,6 +27,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`579` fixes an interaction with `--pdb` and `--trace` and task that return. The debugging modes swallowed the return and `None` was returned. Closes {issue}`574`. - {pull}`581` simplifies the code for tracebacks and unpublishes some utility functions. +- {pull}`582` use typed-settings to parse configuration files and create the CLI. - {pull}`586` improves linting. - {pull}`587` improves typing of `capture.py`. - {pull}`588` resets class variables of `ExecutionReport` and `Traceback`. diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md index 56204c28..72d5e6a9 100644 --- a/docs/source/reference_guides/api.md +++ b/docs/source/reference_guides/api.md @@ -10,7 +10,6 @@ pytask offers the following functionalities. ```{eval-rst} .. autoclass:: pytask.ColoredCommand .. autoclass:: pytask.ColoredGroup -.. autoclass:: pytask.EnumChoice ``` ## Compatibility diff --git a/docs/source/reference_guides/configuration.md b/docs/source/reference_guides/configuration.md index da8ece2b..cf732808 100644 --- a/docs/source/reference_guides/configuration.md +++ b/docs/source/reference_guides/configuration.md @@ -42,23 +42,6 @@ are welcome to also support macOS. ```` -````{confval} database_url - -pytask uses a database to keep track of tasks, products, and dependencies over runs. By -default, it will create an SQLite database in the project's root directory called -`.pytask/pytask.sqlite3`. If you want to use a different name or a different dialect -[supported by sqlalchemy](https://docs.sqlalchemy.org/en/latest/core/engines.html#backend-specific-urls), -use either {option}`pytask build --database-url` or `database_url` in the config. - -```toml -database_url = "sqlite:///.pytask/pytask.sqlite3" -``` - -Relative paths for SQLite databases are interpreted as either relative to the -configuration file or the root directory. - -```` - ````{confval} editor_url_scheme Depending on your terminal, pytask is able to turn task ids into clickable links to the diff --git a/docs/source/tutorials/visualizing_the_dag.md b/docs/source/tutorials/visualizing_the_dag.md index 5797576e..be6f92ce 100644 --- a/docs/source/tutorials/visualizing_the_dag.md +++ b/docs/source/tutorials/visualizing_the_dag.md @@ -5,7 +5,7 @@ To visualize the {term}`DAG` of the project, first, install [graphviz](https://graphviz.org/). For example, you can both install with conda ```console -$ conda install -c conda-forge pygraphviz +conda install -c conda-forge pygraphviz ``` After that, pytask offers two interfaces to visualize your project's {term}`DAG`. @@ -15,7 +15,7 @@ After that, pytask offers two interfaces to visualize your project's {term}`DAG` You can quickly create a visualization with this command. ```console -$ pytask dag +pytask dag ``` It generates a `dag.pdf` in the current working directory. @@ -26,7 +26,7 @@ file-ending. Select any format supported by [graphviz](https://graphviz.org/docs/outputs/). ```console -$ pytask dag -o dag.png +pytask dag -o dag.png ``` You can change the graph's layout by using the {option}`pytask dag --layout` option. Its diff --git a/pyproject.toml b/pyproject.toml index 8a11a25c..af29e7f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "sqlalchemy>=2", 'tomli>=1; python_version < "3.11"', 'typing-extensions; python_version < "3.9"', + "typed-settings[option-groups]", "universal-pathlib>=0.2.2", ] @@ -113,6 +114,9 @@ source = "vcs" [tool.hatch.metadata] allow-direct-references = true +[tool.setuptools_scm] +version_file = "src/_pytask/_version.py" + [tool.ruff] target-version = "py38" fix = true @@ -184,6 +188,7 @@ disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true warn_unused_ignores = true +plugins = ["typed_settings.mypy"] [[tool.mypy.overrides]] module = "tests.*" @@ -194,14 +199,14 @@ ignore_errors = true module = ["click_default_group", "networkx"] ignore_missing_imports = true -[[tool.mypy.overrides]] -module = ["_pytask.coiled_utils"] -disable_error_code = ["import-not-found"] - [[tool.mypy.overrides]] module = ["_pytask.hookspecs"] disable_error_code = ["empty-body"] +[[tool.mypy.overrides]] +module = ["_pytask.coiled_utils"] +disable_error_code = ["import-untyped", "import-not-found"] + [tool.codespell] skip = "*.js,*/termynal.css" @@ -219,3 +224,10 @@ exclude_also = [ [tool.mdformat] wrap = 88 end_of_line = "keep" + + +[tool.pytask] +debug_pytask = 1 + +[tool.pytask.ini_options] +capture = "all" diff --git a/src/_pytask/build.py b/src/_pytask/build.py index ef3fd857..f939e7d6 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -5,20 +5,14 @@ import json import sys from contextlib import suppress -from pathlib import Path from typing import TYPE_CHECKING from typing import Any from typing import Callable from typing import Iterable from typing import Literal -import click +import typed_settings as ts -from _pytask.capture_utils import CaptureMethod -from _pytask.capture_utils import ShowCapture -from _pytask.click import ColoredCommand -from _pytask.config_utils import find_project_root_and_config -from _pytask.config_utils import read_config from _pytask.console import console from _pytask.dag import create_dag from _pytask.exceptions import CollectionError @@ -27,31 +21,75 @@ from _pytask.exceptions import ResolvingDependenciesError from _pytask.outcomes import ExitCode from _pytask.path import HashPathCache -from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.shared import parse_paths +from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import update_settings from _pytask.shared import to_list from _pytask.traceback import Traceback +from _pytask.typing import NoDefault +from _pytask.typing import no_default if TYPE_CHECKING: + from pathlib import Path from typing import NoReturn + from _pytask.capture_utils import CaptureMethod + from _pytask.capture_utils import ShowCapture from _pytask.node_protocols import PTask + from _pytask.settings import Settings + + +@ts.settings +class Build: + stop_after_first_failure: bool = ts.option( + default=False, + click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, + help="Stop after the first failure.", + ) + max_failures: float = ts.option( + default=float("inf"), + click={"param_decls": ("--max-failures",)}, + help="Stop after some failures.", + ) + show_errors_immediately: bool = ts.option( + default=False, + click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, + help="Show errors with tracebacks as soon as the task fails.", + ) + show_traceback: bool = ts.option( + default=True, + click={"param_decls": ("--show-traceback", "--show-no-traceback")}, + help="Choose whether tracebacks should be displayed or not.", + ) + dry_run: bool = ts.option( + default=False, + click={"param_decls": ("--dry-run",), "is_flag": True}, + help="Perform a dry-run.", + ) + force: bool = ts.option( + default=False, + click={"param_decls": ("-f", "--force"), "is_flag": True}, + help="Execute a task even if it succeeded successfully before.", + ) + check_casing_of_paths: bool = ts.option( + default=True, + click={"param_decls": ("--check-casing-of-paths",), "hidden": True}, + ) @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - cli.add_command(build_command) + settings_builder.commands["build"] = build_command + settings_builder.option_groups["build"] = Build() @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Fill cache of file hashes with stored hashes.""" with suppress(Exception): - path = config["root"] / ".pytask" / "file_hashes.json" + path = config.common.cache / "file_hashes.json" cache = json.loads(path.read_text()) for key, value in cache.items(): @@ -61,43 +99,45 @@ def pytask_post_parse(config: dict[str, Any]) -> None: @hookimpl def pytask_unconfigure(session: Session) -> None: """Save calculated file hashes to file.""" - path = session.config["root"] / ".pytask" / "file_hashes.json" + path = session.config.common.cache / "file_hashes.json" path.write_text(json.dumps(HashPathCache._cache)) -def build( # noqa: C901, PLR0912, PLR0913 +def build( # noqa: PLR0913 *, - capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD, - check_casing_of_paths: bool = True, - config: Path | None = None, - database_url: str = "", - debug_pytask: bool = False, - disable_warnings: bool = False, - dry_run: bool = False, + capture: Literal["fd", "no", "sys", "tee-sys"] + | CaptureMethod + | NoDefault = no_default, + check_casing_of_paths: bool | NoDefault = no_default, + debug_pytask: bool | NoDefault = no_default, + disable_warnings: bool | NoDefault = no_default, + dry_run: bool | NoDefault = no_default, editor_url_scheme: Literal["no_link", "file", "vscode", "pycharm"] # noqa: PYI051 - | str = "file", - expression: str = "", - force: bool = False, - ignore: Iterable[str] = (), - marker_expression: str = "", - max_failures: float = float("inf"), - n_entries_in_table: int = 15, - paths: Path | Iterable[Path] = (), - pdb: bool = False, - pdb_cls: str = "", - s: bool = False, + | str + | NoDefault = no_default, + expression: str | NoDefault = no_default, + force: bool | NoDefault = no_default, + ignore: Iterable[str] | NoDefault = no_default, + marker_expression: str | NoDefault = no_default, + max_failures: float | NoDefault = no_default, + n_entries_in_table: int | NoDefault = no_default, + paths: Path | Iterable[Path] | NoDefault = no_default, + pdb: bool | NoDefault = no_default, + pdb_cls: str | NoDefault = no_default, + s: bool | NoDefault = no_default, show_capture: Literal["no", "stdout", "stderr", "all"] - | ShowCapture = ShowCapture.ALL, - show_errors_immediately: bool = False, - show_locals: bool = False, - show_traceback: bool = True, - sort_table: bool = True, - stop_after_first_failure: bool = False, - strict_markers: bool = False, + | ShowCapture + | NoDefault = no_default, + show_errors_immediately: bool | NoDefault = no_default, + show_locals: bool | NoDefault = no_default, + show_traceback: bool | NoDefault = no_default, + sort_table: bool | NoDefault = no_default, + stop_after_first_failure: bool | NoDefault = no_default, + strict_markers: bool | NoDefault = no_default, tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), - task_files: Iterable[str] = ("task_*.py",), - trace: bool = False, - verbose: int = 1, + task_files: Iterable[str] | NoDefault = no_default, + trace: bool | NoDefault = no_default, + verbose: int | NoDefault = no_default, **kwargs: Any, ) -> Session: """Run pytask. @@ -112,10 +152,6 @@ def build( # noqa: C901, PLR0912, PLR0913 The capture method for stdout and stderr. check_casing_of_paths Whether errors should be raised when file names have different casings. - config - A path to the configuration file. - database_url - An URL to the database that tracks the status of tasks. debug_pytask Whether debug information should be shown. disable_warnings @@ -180,11 +216,9 @@ def build( # noqa: C901, PLR0912, PLR0913 """ try: - raw_config = { + updates = { "capture": capture, "check_casing_of_paths": check_casing_of_paths, - "config": config, - "database_url": database_url, "debug_pytask": debug_pytask, "disable_warnings": disable_warnings, "dry_run": dry_run, @@ -195,7 +229,7 @@ def build( # noqa: C901, PLR0912, PLR0913 "marker_expression": marker_expression, "max_failures": max_failures, "n_entries_in_table": n_entries_in_table, - "paths": paths, + "paths": to_list(paths) if paths is not no_default else no_default, "pdb": pdb, "pdb_cls": pdb_cls, "s": s, @@ -212,48 +246,42 @@ def build( # noqa: C901, PLR0912, PLR0913 "verbose": verbose, **kwargs, } + filtered_updates = {k: v for k, v in updates.items() if v is not no_default} - if "command" not in raw_config: - pm = get_plugin_manager() - storage.store(pm) - else: - pm = storage.get() + from _pytask.cli import settings_builder - # If someone called the programmatic interface, we need to do some parsing. - if "command" not in raw_config: - raw_config["command"] = "build" - # Add defaults from cli. - from _pytask.cli import DEFAULTS_FROM_CLI - - raw_config = {**DEFAULTS_FROM_CLI, **raw_config} - - raw_config["paths"] = parse_paths(raw_config["paths"]) + settings = settings_builder.load_settings(kwargs=filtered_updates) + except (ConfigurationError, Exception): + console.print(Traceback(sys.exc_info())) + session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) + else: + session = _internal_build(settings=settings, tasks=tasks) + return session - if raw_config["config"] is not None: - raw_config["config"] = Path(raw_config["config"]).resolve() - raw_config["root"] = raw_config["config"].parent - else: - ( - raw_config["root"], - raw_config["config"], - ) = find_project_root_and_config(raw_config["paths"]) - if raw_config["config"] is not None: - config_from_file = read_config(raw_config["config"]) +def build_command(settings: Any, **arguments: Any) -> NoReturn: + """Collect tasks, execute them and report the results. - if "paths" in config_from_file: - paths = config_from_file["paths"] - paths = [ - raw_config["config"].parent.joinpath(path).resolve() - for path in to_list(paths) - ] - config_from_file["paths"] = paths + The default command. pytask collects tasks from the given paths or the + current working directory, executes them and reports the results. - raw_config = {**raw_config, **config_from_file} + """ + settings = update_settings(settings, arguments) + session = _internal_build(settings=settings) + sys.exit(session.exit_code) - config_ = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config_) +def _internal_build( + settings: Settings, + tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), +) -> Session: + """Run pytask internally.""" + try: + config = settings.common.pm.hook.pytask_configure( + pm=settings.common.pm, config=settings + ) + session = Session(config=config, hook=config.common.pm.hook) + session.attrs["tasks"] = tasks except (ConfigurationError, Exception): console.print(Traceback(sys.exc_info())) @@ -281,57 +309,3 @@ def build( # noqa: C901, PLR0912, PLR0913 session.hook.pytask_unconfigure(session=session) return session - - -@click.command(cls=ColoredCommand, name="build") -@click.option( - "--debug-pytask", - is_flag=True, - default=False, - help="Trace all function calls in the plugin framework.", -) -@click.option( - "-x", - "--stop-after-first-failure", - is_flag=True, - default=False, - help="Stop after the first failure.", -) -@click.option( - "--max-failures", - type=click.FloatRange(min=1), - default=float("inf"), - help="Stop after some failures.", -) -@click.option( - "--show-errors-immediately", - is_flag=True, - default=False, - help="Show errors with tracebacks as soon as the task fails.", -) -@click.option( - "--show-traceback/--show-no-traceback", - type=bool, - default=True, - help="Choose whether tracebacks should be displayed or not.", -) -@click.option( - "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry-run." -) -@click.option( - "-f", - "--force", - is_flag=True, - default=False, - help="Execute a task even if it succeeded successfully before.", -) -def build_command(**raw_config: Any) -> NoReturn: - """Collect tasks, execute them and report the results. - - The default command. pytask collects tasks from the given paths or the - current working directory, executes them and reports the results. - - """ - raw_config["command"] = "build" - session = build(**raw_config) - sys.exit(session.exit_code) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 9e0f1244..b55680cb 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -43,65 +43,64 @@ from typing import TextIO from typing import final -import click +import typed_settings as ts from typing_extensions import Self from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture -from _pytask.click import EnumChoice from _pytask.pluginmanager import hookimpl -from _pytask.shared import convert_to_enum if TYPE_CHECKING: from types import TracebackType from _pytask.node_protocols import PTask + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + + +@ts.settings +class Capture: + """Settings for capturing.""" + + capture: CaptureMethod = ts.option( + default=CaptureMethod.FD, + click={"param_decls": ["--capture"]}, + help="Per task capturing method.", + ) + s: bool = ts.option( + default=False, + click={"param_decls": ["-s"], "is_flag": True}, + help="Shortcut for --capture=no.", + ) + show_capture: ShowCapture = ts.option( + default=ShowCapture.ALL, + click={"param_decls": ["--show-capture"]}, + help="Choose which captured output should be shown for failed tasks.", + ) @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Add CLI options for capturing output.""" - additional_parameters = [ - click.Option( - ["--capture"], - type=EnumChoice(CaptureMethod), - default=CaptureMethod.FD, - help="Per task capturing method.", - ), - click.Option( - ["-s"], - is_flag=True, - default=False, - help="Shortcut for --capture=no.", - ), - click.Option( - ["--show-capture"], - type=EnumChoice(ShowCapture), - default=ShowCapture.ALL, - help="Choose which captured output should be shown for failed tasks.", - ), - ] - cli.commands["build"].params.extend(additional_parameters) + settings_builder.option_groups["capture"] = Capture() @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse configuration. Note that, ``-s`` is a shortcut for ``--capture=no``. """ - config["capture"] = convert_to_enum(config["capture"], CaptureMethod) - if config["s"]: - config["capture"] = CaptureMethod.NO - config["show_capture"] = convert_to_enum(config["show_capture"], ShowCapture) + if config.capture.s: + config.capture.capture = CaptureMethod.NO @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Initialize the CaptureManager.""" - pluginmanager = config["pm"] - capman = CaptureManager(config["capture"]) + pluginmanager = config.common.pm + capman = CaptureManager(config.capture.capture) pluginmanager.register(capman, "capturemanager") capman.stop_capturing() capman.start_capturing() @@ -466,12 +465,9 @@ def __init__(self, targetfd: int) -> None: self._state = "initialized" def __repr__(self) -> str: - return "<{} {} oldfd={} _state={!r} tmpfile={!r}>".format( # noqa: UP032 - self.__class__.__name__, - self.targetfd, - self.targetfd_save, - self._state, - self.tmpfile, + return ( + f"<{self.__class__.__name__} {self.targetfd} oldfd={self.targetfd_save} " + f"_state={self._state!r} tmpfile={self.tmpfile!r}>" ) def _assert_state(self, op: str, states: tuple[str, ...]) -> None: @@ -619,15 +615,9 @@ def __init__( self.err: CaptureBase[AnyStr] | None = err def __repr__(self) -> str: - return ( # noqa: UP032 - "" - ).format( - self.out, - self.err, - self.in_, - self._state, - self._in_suspended, + return ( + f"" ) def start_capturing(self) -> None: @@ -734,8 +724,8 @@ def __init__(self, method: CaptureMethod) -> None: self._capturing: MultiCapture[str] | None = None def __repr__(self) -> str: - return ("").format( # noqa: UP032 - self._method, self._capturing + return ( + f"" ) def is_capturing(self) -> bool: diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index a4980f3e..f3a6bbfe 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -2,21 +2,19 @@ from __future__ import annotations -import enum import itertools import shutil import sys -from pathlib import Path +from enum import Enum from typing import TYPE_CHECKING from typing import Any from typing import Generator from typing import Iterable import click +import typed_settings as ts from attrs import define -from _pytask.click import ColoredCommand -from _pytask.click import EnumChoice from _pytask.console import console from _pytask.exceptions import CollectionError from _pytask.git import get_all_files @@ -29,82 +27,82 @@ from _pytask.path import find_common_ancestor from _pytask.path import relative_to from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.shared import to_list +from _pytask.settings_utils import update_settings from _pytask.traceback import Traceback from _pytask.tree_util import tree_leaves if TYPE_CHECKING: + from pathlib import Path from typing import NoReturn + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder -class _CleanMode(enum.Enum): + +_DEFAULT_EXCLUDE: tuple[str, ...] = (".git/*",) + + +class _CleanMode(Enum): DRY_RUN = "dry-run" FORCE = "force" INTERACTIVE = "interactive" -_DEFAULT_EXCLUDE: list[str] = [".git/*"] - - -_HELP_TEXT_MODE = ( - "Choose 'dry-run' to print the paths of files/directories which would be removed, " - "'interactive' for a confirmation prompt for every path, and 'force' to remove all " - "unknown paths at once." -) +@ts.settings +class Clean: + directories: bool = ts.option( + default=False, + help="Remove whole directories.", + click={"is_flag": True, "param_decls": ["-d", "--directories"]}, + ) + exclude: tuple[str, ...] = ts.option( + factory=tuple, + help="A filename pattern to exclude files from the cleaning process.", + click={ + "multiple": True, + "metavar": "PATTERN", + "param_decls": ["-e", "--exclude"], + }, + ) + mode: _CleanMode = ts.option( + default=_CleanMode.DRY_RUN, + help=( + "Choose 'dry-run' to print the paths of files/directories which would be " + "removed, 'interactive' for a confirmation prompt for every path, and " + "'force' to remove all unknown paths at once." + ), + click={"param_decls": ["-m", "--mode"]}, + ) + quiet: bool = ts.option( + default=False, + help="Do not print the names of the removed paths.", + click={"is_flag": True, "param_decls": ["-q", "--quiet"]}, + ) @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - cli.add_command(clean) + settings_builder.commands["clean"] = clean_command + settings_builder.option_groups["clean"] = Clean() @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" - config["exclude"] = to_list(config["exclude"]) + _DEFAULT_EXCLUDE - - -@click.command(cls=ColoredCommand) -@click.option( - "-d", - "--directories", - is_flag=True, - default=False, - help="Remove whole directories.", -) -@click.option( - "-e", - "--exclude", - metavar="PATTERN", - multiple=True, - type=str, - help="A filename pattern to exclude files from the cleaning process.", -) -@click.option( - "--mode", - default=_CleanMode.DRY_RUN, - type=EnumChoice(_CleanMode), - help=_HELP_TEXT_MODE, -) -@click.option( - "-q", - "--quiet", - is_flag=True, - help="Do not print the names of the removed paths.", - default=False, -) -def clean(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912 + config.clean.exclude = config.clean.exclude + _DEFAULT_EXCLUDE + + +def clean_command(settings: Settings, **arguments: Any) -> NoReturn: # noqa: C901, PLR0912 """Clean the provided paths by removing files unknown to pytask.""" - pm = storage.get() - raw_config["command"] = "clean" + settings = update_settings(settings, arguments) + pm = settings.common.pm try: # Duplication of the same mechanism in :func:`pytask.build`. - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except Exception: # noqa: BLE001 # pragma: no cover session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) @@ -116,32 +114,31 @@ def clean(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912 session.hook.pytask_collect(session=session) known_paths = _collect_all_paths_known_to_pytask(session) - exclude = session.config["exclude"] - include_directories = session.config["directories"] + exclude = session.config.clean.exclude + include_directories = session.config.clean.directories unknown_paths = _find_all_unknown_paths( session, known_paths, exclude, include_directories ) common_ancestor = find_common_ancestor( - *unknown_paths, *session.config["paths"] + *unknown_paths, *session.config.common.paths ) if unknown_paths: targets = "Files" - if session.config["directories"]: + if session.config.clean.directories: targets += " and directories" console.print(f"\n{targets} which can be removed:\n") for path in unknown_paths: short_path = relative_to(path, common_ancestor) - if session.config["mode"] == _CleanMode.DRY_RUN: + if session.config.clean.mode == _CleanMode.DRY_RUN: console.print(f"Would remove {short_path}") else: - should_be_deleted = session.config[ - "mode" - ] == _CleanMode.FORCE or click.confirm( - f"Would you like to remove {short_path}?" + should_be_deleted = ( + session.config.clean.mode == _CleanMode.FORCE + or click.confirm(f"Would you like to remove {short_path}?") ) if should_be_deleted: - if not session.config["quiet"]: + if not session.config.clean.quiet: console.print(f"Remove {short_path}") if path.is_dir(): shutil.rmtree(path) @@ -185,19 +182,17 @@ def _collect_all_paths_known_to_pytask(session: Session) -> set[Path]: known_paths = known_files | known_directories - if session.config["config"]: - known_paths.add(session.config["config"]) - known_paths.add(session.config["root"]) + if session.config.common.config_file: + known_paths.add(session.config.common.config_file) + known_paths.add(session.config.common.root) - database_url = session.config["database_url"] - if database_url.drivername == "sqlite" and database_url.database: - known_paths.add(Path(database_url.database)) + known_paths.add(session.config.common.cache / "pytask.sqlite3") # Add files tracked by git. if is_git_installed(): - git_root = get_root(session.config["root"]) + git_root = get_root(session.config.common.root) if git_root is not None: - paths_known_by_git = get_all_files(session.config["root"]) + paths_known_by_git = get_all_files(session.config.common.root) absolute_paths_known_by_git = [ git_root.joinpath(p) for p in paths_known_by_git ] @@ -231,7 +226,7 @@ def _find_all_unknown_paths( """ recursive_nodes = [ _RecursivePathNode.from_path(path, known_paths, exclude) - for path in session.config["paths"] + for path in session.config.common.paths ] return list( itertools.chain.from_iterable( diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 13046646..30eb5d2d 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -2,13 +2,18 @@ from __future__ import annotations +import sys from typing import Any import click from packaging.version import parse as parse_version +from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup +from _pytask.console import console from _pytask.pluginmanager import storage +from _pytask.settings_utils import SettingsBuilder +from _pytask.traceback import Traceback _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), @@ -22,20 +27,18 @@ _VERSION_OPTION_KWARGS = {} -def _extend_command_line_interface(cli: click.Group) -> click.Group: +def _extend_command_line_interface() -> SettingsBuilder: """Add parameters from plugins to the commandline interface.""" pm = storage.create() - pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": cli}) - _sort_options_for_each_command_alphabetically(cli) - return cli + settings_builder = SettingsBuilder() + pm.hook.pytask_extend_command_line_interface.call_historic( + kwargs={"settings_builder": settings_builder} + ) + return settings_builder -def _sort_options_for_each_command_alphabetically(cli: click.Group) -> None: - """Sort command line options and arguments for each command alphabetically.""" - for command in cli.commands: - cli.commands[command].params = sorted( - cli.commands[command].params, key=lambda x: x.opts[0].replace("-", "") - ) +settings_builder = _extend_command_line_interface() +decorator = settings_builder.build_decorator() @click.group( @@ -49,11 +52,11 @@ def cli() -> None: """Manage your tasks with pytask.""" -_extend_command_line_interface(cli) - - -DEFAULTS_FROM_CLI = { - option.name: option.default - for command in cli.commands.values() - for option in command.params -} +try: + for name, func in settings_builder.commands.items(): + command = click.command(name=name, cls=ColoredCommand)(decorator(func)) + command.params.extend(settings_builder.arguments) + cli.add_command(command) +except Exception: # noqa: BLE001 + traceback = Traceback(sys.exc_info(), show_locals=False) + console.print(traceback) diff --git a/src/_pytask/click.py b/src/_pytask/click.py index 6daab734..36d24d20 100644 --- a/src/_pytask/click.py +++ b/src/_pytask/click.py @@ -11,7 +11,6 @@ from typing import ClassVar import click -from click import Choice from click import Command from click import Context from click import Parameter @@ -29,37 +28,7 @@ from collections.abc import Sequence -__all__ = ["ColoredCommand", "ColoredGroup", "EnumChoice"] - - -class EnumChoice(Choice): - """An enum-based choice type. - - The implementation is copied from https://github.com/pallets/click/pull/2210 and - related discussion can be found in https://github.com/pallets/click/issues/605. - - In contrast to using :class:`click.Choice`, using this type ensures that the error - message does not show the enum members. - - In contrast to the proposed implementation in the PR, this implementation does not - use the members than rather the values of the enum. - - """ - - def __init__(self, enum_type: type[Enum], case_sensitive: bool = True) -> None: - super().__init__( - choices=[element.value for element in enum_type], - case_sensitive=case_sensitive, - ) - self.enum_type = enum_type - - def convert(self, value: Any, param: Parameter | None, ctx: Context | None) -> Any: - if isinstance(value, Enum): - value = value.value - value = super().convert(value=value, param=param, ctx=ctx) - if value is None: - return None - return self.enum_type(value) +__all__ = ["ColoredCommand", "ColoredGroup"] class _OptionHighlighter(RegexHighlighter): @@ -207,7 +176,7 @@ def _print_options(group_or_command: Command | DefaultGroup, ctx: Context) -> No options_table = Table(highlight=True, box=None, show_header=False) - for param in group_or_command.get_params(ctx): + for param in sorted(group_or_command.get_params(ctx), key=lambda x: x.name): if isinstance(param, click.Argument): continue diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 055a80d5..59228657 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -47,7 +47,6 @@ from _pytask.pluginmanager import hookimpl from _pytask.reports import CollectionReport from _pytask.shared import find_duplicates -from _pytask.shared import to_list from _pytask.shared import unwrap_task_function from _pytask.task_utils import COLLECTED_TASKS from _pytask.task_utils import task as task_decorator @@ -56,6 +55,7 @@ if TYPE_CHECKING: from _pytask.models import NodeInfo from _pytask.session import Session + from _pytask.settings import Settings @hookimpl @@ -94,7 +94,7 @@ def _collect_from_paths(session: Session) -> None: Go through all paths, check if the path is ignored, and collect the file if not. """ - for path in _not_ignored_paths(session.config["paths"], session): + for path in _not_ignored_paths(session.config.common.paths, session): reports = session.hook.pytask_collect_file_protocol( session=session, path=path, reports=session.collection_reports ) @@ -105,7 +105,7 @@ def _collect_from_paths(session: Session) -> None: def _collect_from_tasks(session: Session) -> None: """Collect tasks from user provided tasks via the functional interface.""" - for raw_task in to_list(session.config.get("tasks", ())): + for raw_task in session.attrs.get("tasks", []): if is_task_function(raw_task): if not hasattr(raw_task, "pytask_meta"): raw_task = task_decorator()(raw_task) # noqa: PLW2901 @@ -174,9 +174,9 @@ def _collect_not_collected_tasks(session: Session) -> None: @hookimpl -def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: +def pytask_ignore_collect(path: Path, config: Settings) -> bool: """Ignore a path during the collection.""" - return any(path.match(pattern) for pattern in config["ignore"]) + return any(path.match(pattern) for pattern in config.common.ignore) @hookimpl @@ -190,7 +190,7 @@ def pytask_collect_file_protocol( ) flat_reports = list(itertools.chain.from_iterable(new_reports)) except Exception: # noqa: BLE001 - name = shorten_path(path, session.config["paths"]) + name = shorten_path(path, session.config.common.paths) node = PathNode(name=name, path=path) flat_reports = [ CollectionReport.from_exception( @@ -208,8 +208,8 @@ def pytask_collect_file( session: Session, path: Path, reports: list[CollectionReport] ) -> list[CollectionReport] | None: """Collect a file.""" - if any(path.match(pattern) for pattern in session.config["task_files"]): - mod = import_path(path, session.config["root"]) + if any(path.match(pattern) for pattern in session.config.common.task_files): + mod = import_path(path, session.config.common.root) collected_reports = [] for name, obj in inspect.getmembers(mod): @@ -388,7 +388,8 @@ def pytask_collect_node( # noqa: C901, PLR0912 or node.name == node.root_dir.joinpath(node.pattern).as_posix() ): short_root_dir = shorten_path( - node.root_dir, session.config["paths"] or (session.config["root"],) + node.root_dir, + session.config.common.paths or (session.config.common.root,), ) node.name = Path(short_root_dir, node.pattern).as_posix() @@ -408,7 +409,7 @@ def pytask_collect_node( # noqa: C901, PLR0912 # check which will fail since ``.resolves()`` also normalizes a path. node.path = Path(os.path.normpath(node.path)) _raise_error_if_casing_of_path_is_wrong( - node.path, session.config["check_casing_of_paths"] + node.path, session.config.build.check_casing_of_paths ) if isinstance(node, PPathNode) and ( @@ -416,7 +417,7 @@ def pytask_collect_node( # noqa: C901, PLR0912 ): # Shorten name of PathNodes. node.name = shorten_path( - node.path, session.config["paths"] or (session.config["root"],) + node.path, session.config.common.paths or (session.config.common.root,) ) # Skip ``is_dir`` for remote UPaths because it downloads the file and blocks the @@ -447,9 +448,11 @@ def pytask_collect_node( # noqa: C901, PLR0912 # check which will fail since ``.resolves()`` also normalizes a path. node = Path(os.path.normpath(node)) _raise_error_if_casing_of_path_is_wrong( - node, session.config["check_casing_of_paths"] + node, session.config.build.check_casing_of_paths + ) + name = shorten_path( + node, session.config.common.paths or (session.config.common.root,) ) - name = shorten_path(node, session.config["paths"] or (session.config["root"],)) if isinstance(node, Path) and node.is_dir(): raise ValueError(_TEMPLATE_ERROR_DIRECTORY.format(path=node)) diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 2e953e5c..5f8a5fab 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -7,11 +7,10 @@ from typing import TYPE_CHECKING from typing import Any -import click +import typed_settings as ts from rich.text import Text from rich.tree import Tree -from _pytask.click import ColoredCommand from _pytask.console import FILE_ICON from _pytask.console import PYTHON_ICON from _pytask.console import TASK_ICON @@ -32,39 +31,45 @@ from _pytask.path import find_common_ancestor from _pytask.path import relative_to from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings_utils import update_settings from _pytask.tree_util import tree_leaves if TYPE_CHECKING: from pathlib import Path from typing import NoReturn + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + + +@ts.settings +class Collect: + nodes: bool = ts.option( + default=False, + help="Show a task's dependencies and products.", + click={"is_flag": True, "param_decls": ["--nodes"]}, + ) + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - cli.add_command(collect) + settings_builder.commands["collect"] = collect + settings_builder.option_groups["collect"] = Collect() -@click.command(cls=ColoredCommand) -@click.option( - "--nodes", - is_flag=True, - default=False, - help="Show a task's dependencies and products.", -) -def collect(**raw_config: Any | None) -> NoReturn: +def collect(settings: Settings, **arguments: Any) -> NoReturn: """Collect tasks and report information about them.""" - pm = storage.get() - raw_config["command"] = "collect" + settings = update_settings(settings, arguments) + pm = settings.common.pm try: - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover - session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) + session = Session(config=config, exit_code=ExitCode.CONFIGURATION_FAILED) console.print_exception() else: @@ -77,14 +82,16 @@ def collect(**raw_config: Any | None) -> NoReturn: task_with_path = [t for t in tasks if isinstance(t, PTaskWithPath)] common_ancestor = _find_common_ancestor_of_all_nodes( - task_with_path, session.config["paths"], session.config["nodes"] + task_with_path, + session.config.common.paths, + session.config.collect.nodes, ) dictionary = _organize_tasks(task_with_path) if dictionary: _print_collected_tasks( dictionary, - session.config["nodes"], - session.config["editor_url_scheme"], + session.config.collect.nodes, + session.config.common.editor_url_scheme, common_ancestor, ) @@ -117,7 +124,7 @@ def _select_tasks_by_expressions_and_marker(session: Session) -> list[PTask]: def _find_common_ancestor_of_all_nodes( - tasks: list[PTaskWithPath], paths: list[Path], show_nodes: bool + tasks: list[PTaskWithPath], paths: tuple[Path, ...], show_nodes: bool ) -> Path: """Find common ancestor from all nodes and passed paths.""" all_paths = [] diff --git a/src/_pytask/config.py b/src/_pytask/config.py index 86423ac3..1e6f3bf5 100644 --- a/src/_pytask/config.py +++ b/src/_pytask/config.py @@ -5,21 +5,18 @@ import tempfile from pathlib import Path from typing import TYPE_CHECKING -from typing import Any from _pytask.pluginmanager import hookimpl -from _pytask.shared import parse_markers from _pytask.shared import parse_paths -from _pytask.shared import to_list if TYPE_CHECKING: from pluggy import PluginManager + from _pytask.settings import Settings -_IGNORED_FOLDERS: list[str] = [".git/*", ".venv/*"] - -_IGNORED_FILES: list[str] = [ +_IGNORED_FOLDERS: tuple[str, ...] = (".git/*", ".venv/*") +_IGNORED_FILES: tuple[str, ...] = ( ".codecov.yml", ".gitignore", ".pre-commit-config.yaml", @@ -31,13 +28,9 @@ "pyproject.toml", "setup.cfg", "tox.ini", -] - - -_IGNORED_FILES_AND_FOLDERS: list[str] = _IGNORED_FILES + _IGNORED_FOLDERS - - -IGNORED_TEMPORARY_FILES_AND_FOLDERS: list[str] = [ +) +_IGNORED_FILES_AND_FOLDERS: tuple[str, ...] = _IGNORED_FILES + _IGNORED_FOLDERS +IGNORED_TEMPORARY_FILES_AND_FOLDERS: tuple[str, ...] = ( "*.egg-info/*", ".ipynb_checkpoints/*", ".mypy_cache/*", @@ -48,7 +41,7 @@ "build/*", "dist/*", "pytest_cache/*", -] +) def is_file_system_case_sensitive() -> bool: @@ -61,57 +54,43 @@ def is_file_system_case_sensitive() -> bool: @hookimpl -def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]: +def pytask_configure(pm: PluginManager, config: Settings) -> Settings: """Configure pytask.""" - # Add all values by default so that many plugins do not need to copy over values. - config = {"pm": pm, "markers": {}, **raw_config} - config["markers"] = parse_markers(config["markers"]) - pm.hook.pytask_parse_config(config=config) pm.hook.pytask_post_parse(config=config) - return config @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" - config["root"].joinpath(".pytask").mkdir(exist_ok=True, parents=True) + config.common.cache.mkdir(exist_ok=True, parents=True) - config["paths"] = parse_paths(config["paths"]) + config.common.paths = parse_paths(config.common.paths) - config["markers"] = { + config.markers.markers = { "try_first": "Try to execute a task a early as possible.", "try_last": "Try to execute a task a late as possible.", - **config["markers"], + **config.markers.markers, } - config["ignore"] = ( - to_list(config["ignore"]) + config.common.ignore = ( + config.common.ignore + _IGNORED_FILES_AND_FOLDERS + IGNORED_TEMPORARY_FILES_AND_FOLDERS ) - value = config.get("task_files", ["task_*.py"]) - if not isinstance(value, (list, tuple)) or not all( - isinstance(p, str) for p in value - ): - msg = "'task_files' must be a list of patterns." - raise ValueError(msg) - config["task_files"] = value - - if config["stop_after_first_failure"]: - config["max_failures"] = 1 + if config.build.stop_after_first_failure: + config.build.max_failures = 1 - for name in ("check_casing_of_paths",): - config[name] = bool(config.get(name, True)) - - if config["debug_pytask"]: - config["pm"].trace.root.setwriter(print) - config["pm"].enable_tracing() + if config.common.debug_pytask: + config.common.pm.trace.root.setwriter(print) + config.common.pm.enable_tracing() @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Sort markers alphabetically.""" - config["markers"] = {k: config["markers"][k] for k in sorted(config["markers"])} + config.markers.markers = { + k: config.markers.markers[k] for k in sorted(config.markers.markers) + } diff --git a/src/_pytask/config_utils.py b/src/_pytask/config_utils.py index e78934d1..0c92498e 100644 --- a/src/_pytask/config_utils.py +++ b/src/_pytask/config_utils.py @@ -10,61 +10,13 @@ import click -from _pytask.shared import parse_paths - if sys.version_info >= (3, 11): # pragma: no cover import tomllib else: # pragma: no cover import tomli as tomllib -__all__ = ["find_project_root_and_config", "read_config", "set_defaults_from_config"] - - -def set_defaults_from_config( - context: click.Context, - param: click.Parameter, # noqa: ARG001 - value: Any, -) -> Path | None: - """Set the defaults for the command-line interface from the configuration.""" - # pytask will later walk through all configuration hooks, even the ones not related - # to this command. They might expect the defaults coming from their related - # command-line options during parsing. Here, we add their defaults to the - # configuration. - command_option_names = [option.name for option in context.command.params] - commands = context.parent.command.commands # type: ignore[union-attr] - all_defaults_from_cli = { - option.name: option.default - for name, command in commands.items() - for option in command.params - if name != context.info_name and option.name not in command_option_names - } - context.params.update(all_defaults_from_cli) - - if value: - context.params["config"] = value - context.params["root"] = context.params["config"].parent - else: - if not context.params["paths"]: - context.params["paths"] = (Path.cwd(),) - - context.params["paths"] = parse_paths(context.params["paths"]) - ( - context.params["root"], - context.params["config"], - ) = find_project_root_and_config(context.params["paths"]) - - if context.params["config"] is None: - return None - - config_from_file = read_config(context.params["config"]) - - if context.default_map is None: - context.default_map = {} - context.default_map.update(config_from_file) - context.params.update(config_from_file) - - return context.params["config"] +__all__ = ["find_project_root_and_config", "read_config"] def find_project_root_and_config( diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index d8c8b325..823c294e 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -54,7 +54,7 @@ def create_dag_from_session(session: Session) -> nx.DiGraph: """Create a DAG from a session.""" dag = _create_dag_from_tasks(tasks=session.tasks) _check_if_dag_has_cycles(dag) - _check_if_tasks_have_the_same_products(dag, session.config["paths"]) + _check_if_tasks_have_the_same_products(dag, session.config.common.paths) dag = _modify_dag(session=session, dag=dag) select_tasks_by_marks_and_expressions(session=session, dag=dag) return dag @@ -174,7 +174,9 @@ def _format_dictionary_to_tree(dict_: dict[str, list[str]], title: str) -> str: return render_to_string(tree, console=console, strip_styles=True) -def _check_if_tasks_have_the_same_products(dag: nx.DiGraph, paths: list[Path]) -> None: +def _check_if_tasks_have_the_same_products( + dag: nx.DiGraph, paths: tuple[Path, ...] +) -> None: nodes_created_by_multiple_tasks = [] for node in dag.nodes: diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index f5f7accb..0defbdfa 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -5,18 +5,18 @@ import enum import sys from pathlib import Path +from typing import TYPE_CHECKING from typing import Any +from typing import Callable +from typing import Iterable import click import networkx as nx +import typed_settings as ts from rich.text import Text -from _pytask.click import ColoredCommand -from _pytask.click import EnumChoice from _pytask.compat import check_for_optional_program from _pytask.compat import import_optional_dependency -from _pytask.config_utils import find_project_root_and_config -from _pytask.config_utils import read_config from _pytask.console import console from _pytask.dag import create_dag from _pytask.exceptions import CollectionError @@ -27,11 +27,16 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.shared import parse_paths +from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import create_settings_loaders +from _pytask.settings_utils import update_settings from _pytask.shared import reduce_names_of_multiple_nodes -from _pytask.shared import to_list from _pytask.traceback import Traceback +if TYPE_CHECKING: + from _pytask.node_protocols import PTask + from _pytask.settings import Settings + class _RankDirection(enum.Enum): TB = "TB" @@ -40,52 +45,42 @@ class _RankDirection(enum.Enum): RL = "RL" +@ts.settings +class Dag: + layout: str = ts.option( + default="dot", + help="The layout determines the structure of the graph. Here you find an " + "overview of all available layouts: https://graphviz.org/docs/layouts.", + ) + output_path: Path = ts.option( + click={ + "type": click.Path(file_okay=True, dir_okay=False, path_type=Path), + "param_decls": ["-o", "--output-path"], + }, + default=Path("dag.pdf"), + help="The output path of the visualization. The format is inferred from the " + "file extension.", + ) + rank_direction: _RankDirection = ts.option( + default=_RankDirection.TB, + help="The direction of the directed graph. It can be ordered from top to " + "bottom, TB, left to right, LR, bottom to top, BT, or right to left, RL.", + click={"param_decls": ["-r", "--rank-direction"]}, + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - cli.add_command(dag) - - -_HELP_TEXT_LAYOUT: str = ( - "The layout determines the structure of the graph. Here you find an overview of " - "all available layouts: https://graphviz.org/docs/layouts." -) - - -_HELP_TEXT_OUTPUT: str = ( - "The output path of the visualization. The format is inferred from the file " - "extension." -) - - -_HELP_TEXT_RANK_DIRECTION: str = ( - "The direction of the directed graph. It can be ordered from top to bottom, TB, " - "left to right, LR, bottom to top, BT, or right to left, RL." -) - - -@click.command(cls=ColoredCommand) -@click.option("-l", "--layout", type=str, default="dot", help=_HELP_TEXT_LAYOUT) -@click.option( - "-o", - "--output-path", - type=click.Path(file_okay=True, dir_okay=False, path_type=Path, resolve_path=True), - default="dag.pdf", - help=_HELP_TEXT_OUTPUT, -) -@click.option( - "-r", - "--rank-direction", - type=EnumChoice(_RankDirection), - help=_HELP_TEXT_RANK_DIRECTION, - default=_RankDirection.TB, -) + settings_builder.commands["dag"] = dag + + def dag(**raw_config: Any) -> int: """Create a visualization of the project's directed acyclic graph.""" try: pm = storage.get() config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover console.print_exception() @@ -96,14 +91,14 @@ def dag(**raw_config: Any) -> int: session.hook.pytask_log_session_header(session=session) import_optional_dependency("pygraphviz") check_for_optional_program( - session.config["layout"], + session.config.dag.layout, extra="The layout program is part of the graphviz package which you " "can install with conda.", ) session.hook.pytask_collect(session=session) session.dag = create_dag(session=session) dag = _refine_dag(session) - _write_graph(dag, session.config["output_path"], session.config["layout"]) + _write_graph(dag, session.config.dag.output_path, session.config.dag.layout) except CollectionError: # pragma: no cover session.exit_code = ExitCode.COLLECTION_FAILED @@ -121,7 +116,13 @@ def dag(**raw_config: Any) -> int: sys.exit(session.exit_code) -def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: +def build_dag( + paths: Path | tuple[Path, ...] = (), + tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), + task_files: Iterable[str] = ("task_*.py",), + settings: Settings | None = None, + **kwargs: Any, +) -> nx.DiGraph: """Build the DAG. This function is the programmatic interface to ``pytask dag`` and returns a @@ -131,12 +132,6 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: To change the style of the graph, it might be easier to convert the graph back to networkx, set attributes, and convert back to pygraphviz. - Parameters - ---------- - raw_config : Dict[str, Any] - The configuration usually received from the CLI. For example, use ``{"paths": - "example-directory/"}`` to collect tasks from a directory. - Returns ------- pygraphviz.AGraph @@ -144,44 +139,28 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: """ try: - pm = get_plugin_manager() - storage.store(pm) - - # If someone called the programmatic interface, we need to do some parsing. - if "command" not in raw_config: - raw_config["command"] = "dag" - # Add defaults from cli. - from _pytask.cli import DEFAULTS_FROM_CLI + updates = { + "paths": paths, + "tasks": tasks, + "task_files": task_files, + **kwargs, + } - raw_config = {**DEFAULTS_FROM_CLI, **raw_config} + if settings is None: + from _pytask.cli import settings_builder - raw_config["paths"] = parse_paths(raw_config["paths"]) + pm = get_plugin_manager() + storage.store(pm) - if raw_config["config"] is not None: - raw_config["config"] = Path(raw_config["config"]).resolve() - raw_config["root"] = raw_config["config"].parent - else: - ( - raw_config["root"], - raw_config["config"], - ) = find_project_root_and_config(raw_config["paths"]) - - if raw_config["config"] is not None: - config_from_file = read_config(raw_config["config"]) - - if "paths" in config_from_file: - paths = config_from_file["paths"] - paths = [ - raw_config["config"].parent.joinpath(path).resolve() - for path in to_list(paths) - ] - config_from_file["paths"] = paths - - raw_config = {**raw_config, **config_from_file} - - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) + settings = ts.load_settings( + settings_builder["dag"].build_settings(), create_settings_loaders() + ) + else: + pm = storage.get() - session = Session.from_config(config) + settings = update_settings(settings, updates) + config_ = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session.from_config(config_) except (ConfigurationError, Exception): # pragma: no cover console.print_exception() @@ -191,7 +170,7 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: session.hook.pytask_log_session_header(session=session) import_optional_dependency("pygraphviz") check_for_optional_program( - session.config["layout"], + session.config.dag.layout, extra="The layout program is part of the graphviz package that you " "can install with conda.", ) @@ -203,15 +182,15 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: def _refine_dag(session: Session) -> nx.DiGraph: """Refine the dag for plotting.""" - dag = _shorten_node_labels(session.dag, session.config["paths"]) + dag = _shorten_node_labels(session.dag, session.config.common.paths) dag = _clean_dag(dag) dag = _style_dag(dag) - dag.graph["graph"] = {"rankdir": session.config["rank_direction"].name} + dag.graph["graph"] = {"rankdir": session.config.dag.rank_direction.name} return dag -def _shorten_node_labels(dag: nx.DiGraph, paths: list[Path]) -> nx.DiGraph: +def _shorten_node_labels(dag: nx.DiGraph, paths: tuple[Path, ...]) -> nx.DiGraph: """Shorten the node labels in the graph for a better experience.""" node_names = dag.nodes short_names = reduce_names_of_multiple_nodes(node_names, dag, paths) diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py index 8a9a08cd..23a33f03 100644 --- a/src/_pytask/data_catalog.py +++ b/src/_pytask/data_catalog.py @@ -11,6 +11,7 @@ import pickle import re from pathlib import Path +from typing import TYPE_CHECKING from typing import Any from attrs import define @@ -26,6 +27,9 @@ from _pytask.pluginmanager import storage from _pytask.session import Session +if TYPE_CHECKING: + from _pytask.settings import Settings + __all__ = ["DataCatalog"] @@ -63,7 +67,7 @@ class DataCatalog: path: Path | None = None _entries: dict[str, PNode | PProvisionalNode] = field(factory=dict) _instance_path: Path = field(factory=_get_parent_path_of_data_catalog_module) - _session_config: dict[str, Any] = field( + _session_config: Settings = field( factory=lambda *x: {"check_casing_of_paths": True} # noqa: ARG005 ) diff --git a/src/_pytask/database.py b/src/_pytask/database.py index dfdf96b3..eb43a006 100644 --- a/src/_pytask/database.py +++ b/src/_pytask/database.py @@ -2,44 +2,20 @@ from __future__ import annotations -from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING from sqlalchemy.engine import make_url from _pytask.database_utils import create_database from _pytask.pluginmanager import hookimpl - -@hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: - """Parse the configuration.""" - # Set default. - if not config["database_url"]: - config["database_url"] = make_url( - f"sqlite:///{config['root'].joinpath('.pytask').as_posix()}/pytask.sqlite3" - ) - - if ( - config["database_url"].drivername == "sqlite" - and config["database_url"].database - ) and not Path(config["database_url"].database).is_absolute(): - if config["config"]: - full_path = ( - config["config"] - .parent.joinpath(config["database_url"].database) - .resolve() - ) - else: - full_path = ( - config["root"].joinpath(config["database_url"].database).resolve() - ) - config["database_url"] = config["database_url"]._replace( - database=full_path.as_posix() - ) +if TYPE_CHECKING: + from _pytask.settings import Settings @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post-parse the configuration.""" - create_database(config["database_url"]) + path = config.common.root.joinpath(".pytask").as_posix() + url = make_url(f"sqlite:///{path}/pytask.sqlite3") + create_database(url) diff --git a/src/_pytask/database_utils.py b/src/_pytask/database_utils.py index dc262967..cc14980c 100644 --- a/src/_pytask/database_utils.py +++ b/src/_pytask/database_utils.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING +from sqlalchemy import URL from sqlalchemy import create_engine from sqlalchemy.orm import DeclarativeBase from sqlalchemy.orm import Mapped @@ -43,7 +44,7 @@ class State(BaseTable): hash_: Mapped[str] -def create_database(url: str) -> None: +def create_database(url: URL) -> None: """Create the database.""" engine = create_engine(url) BaseTable.metadata.create_all(bind=engine) diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index bd0b32fe..9499bc01 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -11,6 +11,7 @@ from typing import Generator import click +import typed_settings as ts from _pytask.console import console from _pytask.node_protocols import PTask @@ -27,37 +28,8 @@ from _pytask.capture import CaptureManager from _pytask.live import LiveManager from _pytask.session import Session - - -@hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: - """Extend command line interface.""" - additional_parameters = [ - click.Option( - ["--pdb"], - help="Start the interactive debugger on errors.", - is_flag=True, - default=False, - ), - click.Option( - ["--trace"], - help="Enter debugger in the beginning of each task.", - is_flag=True, - default=False, - ), - click.Option( - ["--pdbcls"], - help=( - "Start a custom debugger on errors. For example: " - "--pdbcls=IPython.terminal.debugger:TerminalPdb" - ), - type=str, - default=None, - metavar="module_name:class_name", - callback=_pdbcls_callback, - ), - ] - cli.commands["build"].params.extend(additional_parameters) + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder def _pdbcls_callback( @@ -67,36 +39,67 @@ def _pdbcls_callback( ) -> tuple[str, str] | None: """Validate the debugger class string passed to pdbcls.""" message = "'pdbcls' must be like IPython.terminal.debugger:TerminalPdb" - if value is None: return None if isinstance(value, str): split = value.split(":") if len(split) != 2: # noqa: PLR2004 raise click.BadParameter(message) - return tuple(split) # type: ignore[return-value] + return (split[0], split[1]) raise click.BadParameter(message) +@ts.settings +class Debugging: + pdb: bool = ts.option( + default=False, + click={"param_decls": ("--pdb",)}, + help="Start the interactive debugger on errors.", + ) + pdbcls: tuple[str, str] | None = ts.option( + default=None, + click={ + "param_decls": ("--pdb-cls",), + "metavar": "module_name:class_name", + "callback": _pdbcls_callback, + }, + help=( + "Start a custom debugger on errors. For example: " + "--pdbcls=IPython.terminal.debugger:TerminalPdb" + ), + ) + trace: bool = ts.option( + default=False, + click={"param_decls": ("--trace",)}, + help="Enter debugger in the beginning of each task.", + ) + + +@hookimpl +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: + """Extend command line interface.""" + settings_builder.option_groups["debugging"] = Debugging() + + @hookimpl(trylast=True) -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post parse the configuration. Register the plugins in this step to let other plugins influence the pdb or trace option and may be disable it. Especially thinking about pytask-parallel. """ - if config["pdb"]: - config["pm"].register(PdbDebugger) + if config.debugging.pdb: + config.common.pm.register(PdbDebugger) - if config["trace"]: - config["pm"].register(PdbTrace) + if config.debugging.trace: + config.common.pm.register(PdbTrace) PytaskPDB._saved.append( (pdb.set_trace, PytaskPDB._pluginmanager, PytaskPDB._config) ) pdb.set_trace = PytaskPDB.set_trace - PytaskPDB._pluginmanager = config["pm"] + PytaskPDB._pluginmanager = config.common.pm PytaskPDB._config = config @@ -115,10 +118,10 @@ class PytaskPDB: """Pseudo PDB that defers to the real pdb.""" _pluginmanager: PluginManager | None = None - _config: dict[str, Any] | None = None + _config: Settings | None = None _saved: ClassVar[list[tuple[Any, ...]]] = [] _recursive_debug: int = 0 - _wrapped_pdb_cls: tuple[type[pdb.Pdb], type[pdb.Pdb]] | None = None + _wrapped_pdb_cls: tuple[tuple[str, str] | None, type[pdb.Pdb]] | None = None @classmethod def _is_capturing(cls, capman: CaptureManager) -> bool: @@ -138,7 +141,7 @@ def _import_pdb_cls( # Happens when using pytask.set_trace outside of a task. return pdb.Pdb - usepdb_cls = cls._config["pdbcls"] + usepdb_cls = cls._config.debugging.pdbcls if cls._wrapped_pdb_cls and cls._wrapped_pdb_cls[0] == usepdb_cls: return cls._wrapped_pdb_cls[1] @@ -329,8 +332,12 @@ def wrap_function_for_post_mortem_debugging(session: Session, task: PTask) -> No @functools.wraps(task_function) def wrapper(*args: Any, **kwargs: Any) -> None: - capman = session.config["pm"].get_plugin("capturemanager") - live_manager = session.config["pm"].get_plugin("live_manager") + capman = session.config.common.pm.get_plugin("capturemanager") + live_manager = session.config.common.pm.get_plugin("live_manager") + + assert capman + assert live_manager + try: return task_function(*args, **kwargs) @@ -393,8 +400,11 @@ def wrap_function_for_tracing(session: Session, task: PTask) -> None: # of the kwargs to task_function was called `func`. @functools.wraps(task_function) def wrapper(*args: Any, **kwargs: Any) -> None: - capman = session.config["pm"].get_plugin("capturemanager") - live_manager = session.config["pm"].get_plugin("live_manager") + capman = session.config.common.pm.get_plugin("capturemanager") + live_manager = session.config.common.pm.get_plugin("live_manager") + + assert capman + assert live_manager # Order is important! Pausing the live object before the capturemanager would # flush the table to stdout and it will be visible in the captured output. diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 039c7f9b..0135bcb9 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -47,13 +47,14 @@ if TYPE_CHECKING: from _pytask.session import Session + from _pytask.settings import Settings @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Adjust the configuration after intermediate values have been parsed.""" - if config["show_errors_immediately"]: - config["pm"].register(ShowErrorsImmediatelyPlugin) + if config.build.show_errors_immediately: + config.common.pm.register(ShowErrorsImmediatelyPlugin) @hookimpl @@ -132,7 +133,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C # Task generators are always executed since their states are not updated, but we # skip the checks as well. - needs_to_be_executed = session.config["force"] or is_task_generator(task) + needs_to_be_executed = session.config.build.force or is_task_generator(task) if not needs_to_be_executed: predecessors = set(dag.predecessors(task.signature)) | {task.signature} @@ -186,7 +187,7 @@ def _safe_load(node: PNode | PProvisionalNode, task: PTask, *, is_product: bool) @hookimpl(trylast=True) def pytask_execute_task(session: Session, task: PTask) -> bool: """Execute task.""" - if session.config["dry_run"]: + if session.config.build.dry_run: raise WouldBeExecuted parameters = inspect.signature(task.function).parameters @@ -234,7 +235,7 @@ def pytask_execute_task_teardown(session: Session, task: PTask) -> None: collect_provisional_products(session, task) missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()] if missing_nodes: - paths = session.config["paths"] + paths = session.config.common.paths files = [format_node_name(i, paths).plain for i in missing_nodes] formatted = format_strings_as_flat_tree( files, "The task did not produce the following files:\n" @@ -279,7 +280,7 @@ def pytask_execute_task_process_report( ) session.n_tasks_failed += 1 - if session.n_tasks_failed >= session.config["max_failures"]: + if session.n_tasks_failed >= session.config.build.max_failures: session.should_stop = True if report.exc_info and isinstance(report.exc_info[1], Exit): # pragma: no cover @@ -292,7 +293,7 @@ def pytask_execute_task_process_report( def pytask_execute_task_log_end(session: Session, report: ExecutionReport) -> None: """Log task outcome.""" url_style = create_url_style_for_task( - report.task.function, session.config["editor_url_scheme"] + report.task.function, session.config.common.editor_url_scheme ) console.print( report.outcome.symbol, @@ -319,7 +320,7 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> counts = count_outcomes(reports, TaskOutcome) - if session.config["show_traceback"]: + if session.config.build.show_traceback: console.print() if counts[TaskOutcome.FAIL]: console.rule( @@ -331,7 +332,7 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> for report in reports: if report.outcome == TaskOutcome.FAIL or ( report.outcome == TaskOutcome.SKIP_PREVIOUS_FAILED - and session.config["verbose"] >= 2 # noqa: PLR2004 + and session.config.common.verbose >= 2 # noqa: PLR2004 ): console.print(report) diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 02ce0721..90ef7c6b 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -15,7 +15,6 @@ if TYPE_CHECKING: from pathlib import Path - import click from pluggy import PluginManager from _pytask.models import NodeInfo @@ -27,6 +26,8 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder hookspec = pluggy.HookspecMarker("pytask") @@ -49,7 +50,7 @@ def pytask_add_hooks(pm: PluginManager) -> None: @hookspec(historic=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface. The hook can be used to extend the command line interface either by providing new @@ -67,7 +68,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: @hookspec(firstresult=True) -def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]: +def pytask_configure(pm: PluginManager, config: Settings) -> Settings: """Configure pytask. The main hook implementation which controls the configuration and calls subordinated @@ -77,12 +78,12 @@ def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, @hookspec -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse configuration that is from CLI or file.""" @hookspec -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post parsing. This hook allows to consolidate the configuration in case some plugins might be @@ -117,7 +118,7 @@ def pytask_collect(session: Session) -> Any: @hookspec(firstresult=True) -def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: +def pytask_ignore_collect(path: Path, config: Settings) -> bool: """Ignore collected path. This hook is indicates for each directory and file whether it should be ignored. diff --git a/src/_pytask/live.py b/src/_pytask/live.py index d7686648..d5f50039 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -8,11 +8,11 @@ from typing import Generator from typing import NamedTuple -import click +import typed_settings as ts from attrs import define from attrs import field from rich.box import ROUNDED -from rich.live import Live +from rich.live import Live as RichLive from rich.status import Status from rich.style import Style from rich.table import Table @@ -30,53 +30,57 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + + +@ts.settings +class Live: + """Settings for live display during the execution.""" + + n_entries_in_table: int = ts.option( + default=15, + click={"param_decls": ["--n-entries-in-table"]}, + help="How many entries to display in the table during the execution. " + "Tasks which are running are always displayed.", + ) + sort_table: bool = ts.option( + default=True, + click={"param_decls": ["--sort-table", "--do-not-sort-table"]}, + help="Sort the table of tasks at the end of the execution.", + ) @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend command line interface.""" - additional_parameters = [ - click.Option( - ["--n-entries-in-table"], - default=15, - type=click.IntRange(min=0), - help="How many entries to display in the table during the execution. " - "Tasks which are running are always displayed.", - ), - click.Option( - ["--sort-table/--do-not-sort-table"], - default=True, - type=bool, - help="Sort the table of tasks at the end of the execution.", - ), - ] - cli.commands["build"].params.extend(additional_parameters) + settings_builder.option_groups["live"] = Live() @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post-parse the configuration.""" live_manager = LiveManager() - config["pm"].register(live_manager, "live_manager") + config.common.pm.register(live_manager, "live_manager") - if config["verbose"] >= 1: + if config.common.verbose >= 1: live_execution = LiveExecution( live_manager=live_manager, - n_entries_in_table=config["n_entries_in_table"], - verbose=config["verbose"], - editor_url_scheme=config["editor_url_scheme"], - sort_final_table=config["sort_table"], + n_entries_in_table=config.live.n_entries_in_table, + verbose=config.common.verbose, + editor_url_scheme=config.common.editor_url_scheme, + sort_final_table=config.live.sort_table, ) - config["pm"].register(live_execution, "live_execution") + config.common.pm.register(live_execution, "live_execution") live_collection = LiveCollection(live_manager=live_manager) - config["pm"].register(live_collection, "live_collection") + config.common.pm.register(live_collection, "live_collection") @hookimpl(wrapper=True) def pytask_execute(session: Session) -> Generator[None, None, None]: - if session.config["verbose"] >= 1: - live_execution = session.config["pm"].get_plugin("live_execution") + if session.config.common.verbose >= 1: + live_execution = session.config.common.pm.get_plugin("live_execution") if live_execution: live_execution.n_tasks = len(session.tasks) return (yield) @@ -102,7 +106,7 @@ class LiveManager: """ - _live = Live(renderable=None, console=console, auto_refresh=False) + _live = RichLive(renderable=None, console=console, auto_refresh=False) def start(self) -> None: self._live.start() diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index bd6ea4dd..5e9cf87f 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -10,8 +10,8 @@ from typing import Any from typing import NamedTuple -import click import pluggy +import typed_settings as ts from rich.text import Text import _pytask @@ -28,6 +28,8 @@ from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.session import Session + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder with contextlib.suppress(ImportError): @@ -41,22 +43,30 @@ class _TimeUnit(NamedTuple): in_seconds: int -@hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: - show_locals_option = click.Option( - ["--show-locals"], - is_flag=True, +@ts.settings +class Logging: + """Settings for logging.""" + + show_locals: bool = ts.option( default=False, + click={"param_decls": ["--show-locals"], "is_flag": True}, help="Show local variables in tracebacks.", ) - cli.commands["build"].params.append(show_locals_option) @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: + settings_builder.option_groups["logging"] = Logging() + + +@hookimpl +def pytask_parse_config(config: Settings) -> None: """Parse configuration.""" - if config["editor_url_scheme"] not in ("no_link", "file") and IS_WINDOWS_TERMINAL: - config["editor_url_scheme"] = "file" + if ( + config.common.editor_url_scheme not in ("no_link", "file") + and IS_WINDOWS_TERMINAL + ): + config.common.editor_url_scheme = "file" warnings.warn( "Windows Terminal does not support url schemes to applications, yet." "See https://github.com/pytask-dev/pytask/issues/171 for more information. " @@ -66,13 +76,13 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: # Set class variables on traceback object. - Traceback._show_locals = config["show_locals"] + Traceback._show_locals = config.logging.show_locals # Set class variables on Executionreport. - ExecutionReport.editor_url_scheme = config["editor_url_scheme"] - ExecutionReport.show_capture = config["show_capture"] - ExecutionReport.show_locals = config["show_locals"] + ExecutionReport.editor_url_scheme = config.common.editor_url_scheme + ExecutionReport.show_capture = config.capture.show_capture + ExecutionReport.show_locals = config.logging.show_locals @hookimpl @@ -83,11 +93,11 @@ def pytask_log_session_header(session: Session) -> None: f"Platform: {sys.platform} -- Python {platform.python_version()}, " f"pytask {_pytask.__version__}, pluggy {pluggy.__version__}" ) - console.print(f"Root: {session.config['root']}") - if session.config["config"] is not None: - console.print(f"Configuration: {session.config['config']}") + console.print(f"Root: {session.config.common.root}") + if session.config.common.config_file is not None: + console.print(f"Configuration: {session.config.common.config_file}") - plugin_info = session.config["pm"].list_plugin_distinfo() + plugin_info = session.config.common.pm.list_plugin_distinfo() if plugin_info: formatted_plugins_w_versions = ", ".join( _format_plugin_names_and_versions(plugin_info) @@ -96,7 +106,7 @@ def pytask_log_session_header(session: Session) -> None: def _format_plugin_names_and_versions( - plugininfo: list[tuple[str, DistFacade]], + plugininfo: list[tuple[Any, DistFacade]], ) -> list[str]: """Format name and version of loaded plugins.""" values: list[str] = [] diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 84e5d387..ff682cbb 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -7,11 +7,10 @@ from typing import AbstractSet from typing import Any -import click +import typed_settings as ts from attrs import define from rich.table import Table -from _pytask.click import ColoredCommand from _pytask.console import console from _pytask.dag_utils import task_and_preceding_tasks from _pytask.exceptions import ConfigurationError @@ -23,8 +22,8 @@ from _pytask.mark.structures import MarkGenerator from _pytask.outcomes import ExitCode from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings_utils import update_settings from _pytask.shared import parse_markers if TYPE_CHECKING: @@ -33,6 +32,8 @@ import networkx as nx from _pytask.node_protocols import PTask + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder __all__ = [ @@ -49,15 +50,39 @@ ] -@click.command(cls=ColoredCommand) -def markers(**raw_config: Any) -> NoReturn: +@ts.settings +class Markers: + """Settings for markers.""" + + strict_markers: bool = ts.option( + default=False, + click={"param_decls": ["--strict-markers"], "is_flag": True}, + help="Raise errors for unknown markers.", + ) + markers: dict[str, str] = ts.option(factory=dict, click={"hidden": True}) + marker_expression: str = ts.option( + default="", + click={ + "param_decls": ["-m", "marker_expression"], + "metavar": "MARKER_EXPRESSION", + }, + help="Select tasks via marker expressions.", + ) + expression: str = ts.option( + default="", + click={"param_decls": ["-k", "expression"], "metavar": "EXPRESSION"}, + help="Select tasks via expressions on task ids.", + ) + + +def markers_command(settings: Settings, **arguments: Any) -> NoReturn: """Show all registered markers.""" - raw_config["command"] = "markers" - pm = storage.get() + settings = update_settings(settings, arguments) + pm = settings.common.pm try: - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover console.print_exception() @@ -66,7 +91,7 @@ def markers(**raw_config: Any) -> NoReturn: else: table = Table("Marker", "Description", leading=1) - for name, description in config["markers"].items(): + for name, description in config.markers.markers.items(): table.add_row(f"pytask.mark.{name}", description) console.print(table) @@ -76,43 +101,21 @@ def markers(**raw_config: Any) -> NoReturn: @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Add marker related options.""" - cli.add_command(markers) - - additional_build_parameters = [ - click.Option( - ["--strict-markers"], - is_flag=True, - help="Raise errors for unknown markers.", - default=False, - ), - click.Option( - ["-m", "marker_expression"], - metavar="MARKER_EXPRESSION", - type=str, - help="Select tasks via marker expressions.", - ), - click.Option( - ["-k", "expression"], - metavar="EXPRESSION", - type=str, - help="Select tasks via expressions on task ids.", - ), - ] - for command in ("build", "clean", "collect"): - cli.commands[command].params.extend(additional_build_parameters) + settings_builder.commands["markers"] = markers_command + settings_builder.option_groups["markers"] = Markers() @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse marker related options.""" MARK_GEN.config = config @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: - config["markers"] = parse_markers(config["markers"]) +def pytask_post_parse(config: Settings) -> None: + config.markers.markers = parse_markers(config.markers.markers) @define(slots=True) @@ -153,7 +156,7 @@ def __call__(self, subname: str) -> bool: def select_by_keyword(session: Session, dag: nx.DiGraph) -> set[str] | None: """Deselect tests by keywords.""" - keywordexpr = session.config["expression"] + keywordexpr = session.config.markers.expression if not keywordexpr: return None @@ -208,7 +211,7 @@ def __call__(self, name: str) -> bool: def select_by_mark(session: Session, dag: nx.DiGraph) -> set[str] | None: """Deselect tests by marks.""" - matchexpr = session.config["marker_expression"] + matchexpr = session.config.markers.marker_expression if not matchexpr: return None diff --git a/src/_pytask/mark/structures.py b/src/_pytask/mark/structures.py index a448a5e4..2939ac04 100644 --- a/src/_pytask/mark/structures.py +++ b/src/_pytask/mark/structures.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from typing import TYPE_CHECKING from typing import Any from typing import Callable from typing import Iterable @@ -14,6 +15,9 @@ from _pytask.models import CollectionMetadata from _pytask.typing import is_task_function +if TYPE_CHECKING: + from _pytask.settings import Settings + @define(frozen=True) class Mark: @@ -186,8 +190,7 @@ class MarkGenerator: """ - config: dict[str, Any] | None = None - """Optional[Dict[str, Any]]: The configuration.""" + config: Settings | None = None def __getattr__(self, name: str) -> MarkDecorator | Any: if name[0] == "_": @@ -206,7 +209,7 @@ def __getattr__(self, name: str) -> MarkDecorator | Any: # If the name is not in the set of known marks after updating, # then it really is time to issue a warning or an error. - if self.config is not None and name not in self.config["markers"]: + if self.config is not None and name not in self.config.markers.markers: if name in ("parametrize", "parameterize", "parametrise", "parameterise"): msg = ( "@pytask.mark.parametrize has been removed since pytask v0.4. " @@ -215,7 +218,7 @@ def __getattr__(self, name: str) -> MarkDecorator | Any: ) raise NotImplementedError(msg) from None - if self.config["strict_markers"]: + if self.config.markers.strict_markers: msg = f"Unknown pytask.mark.{name}." raise ValueError(msg) diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index c3e32f0b..984ba396 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -3,115 +3,24 @@ from __future__ import annotations import importlib.util +import os from pathlib import Path from typing import TYPE_CHECKING from typing import Iterable import click +import typed_settings as ts from click import Context -from sqlalchemy.engine import URL -from sqlalchemy.engine import make_url -from sqlalchemy.exc import ArgumentError +from pluggy import PluginManager # noqa: TCH002 -from _pytask.config_utils import set_defaults_from_config from _pytask.path import import_path +from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import register_hook_impls_from_modules from _pytask.pluginmanager import storage if TYPE_CHECKING: - from pluggy import PluginManager - - -_CONFIG_OPTION = click.Option( - ["-c", "--config"], - callback=set_defaults_from_config, - is_eager=True, - expose_value=False, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - allow_dash=False, - path_type=Path, - resolve_path=True, - ), - help="Path to configuration file.", -) -"""click.Option: An option for the --config flag.""" - - -_IGNORE_OPTION = click.Option( - ["--ignore"], - type=str, - multiple=True, - help=( - "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " - "for more info." - ), - default=[], -) -"""click.Option: An option for the --ignore flag.""" - - -_PATH_ARGUMENT = click.Argument( - ["paths"], - nargs=-1, - type=click.Path(exists=True, resolve_path=True, path_type=Path), - is_eager=True, -) -"""click.Argument: An argument for paths.""" - - -_VERBOSE_OPTION = click.Option( - ["-v", "--verbose"], - type=click.IntRange(0, 2), - default=1, - help="Make pytask verbose (>= 0) or quiet (= 0).", -) -"""click.Option: An option to control pytask's verbosity.""" - - -_EDITOR_URL_SCHEME_OPTION = click.Option( - ["--editor-url-scheme"], - default="file", - help=( - "Use file, vscode, pycharm or a custom url scheme to add URLs to task " - "ids to quickly jump to the task definition. Use no_link to disable URLs." - ), -) -"""click.Option: An option to embed URLs in task ids.""" - - -def _database_url_callback( - ctx: Context, # noqa: ARG001 - name: str, # noqa: ARG001 - value: str | None, -) -> URL | None: - """Check the url for the database.""" - # Since sqlalchemy v2.0.19, we need to shortcircuit here. - if value is None: - return None - - try: - return make_url(value) - except ArgumentError: - msg = ( - "The 'database_url' must conform to sqlalchemy's url standard: " - "https://docs.sqlalchemy.org/en/latest/core/engines.html#backend-specific-urls." - ) - raise click.BadParameter(msg) from None - - -_DATABASE_URL_OPTION = click.Option( - ["--database-url"], - type=str, - help="Url to the database.", - default=None, - show_default="sqlite:///.../.pytask/pytask.sqlite3", - callback=_database_url_callback, -) + from _pytask.settings_utils import SettingsBuilder def _hook_module_callback( @@ -168,26 +77,107 @@ def pytask_add_hooks(pm: PluginManager) -> None: return parsed_modules -_HOOK_MODULE_OPTION = click.Option( - ["--hook-module"], - type=str, - help="Path to a Python module that contains hook implementations.", - multiple=True, - is_eager=True, - callback=_hook_module_callback, +def _path_callback( + ctx: Context, # noqa: ARG001 + param: click.Parameter, # noqa: ARG001 + value: tuple[Path, ...], +) -> tuple[Path, ...]: + """Convert paths to Path objects.""" + return value or (Path.cwd(),) + + +@ts.settings +class Common: + """Common settings for the command line interface.""" + + cache: Path = ts.option(init=False, click={"hidden": True}) + config_file: Path | None = ts.option( + default=None, click={"param_decls": ["--config"], "hidden": True} + ) + debug_pytask: bool = ts.option( + default=False, + click={"param_decls": ("--debug-pytask",), "is_flag": True}, + help="Trace all function calls in the plugin framework.", + ) + editor_url_scheme: str = ts.option( + default="file", + click={"param_decls": ["--editor-url-scheme"]}, + help=( + "Use file, vscode, pycharm or a custom url scheme to add URLs to task " + "ids to quickly jump to the task definition. Use no_link to disable URLs." + ), + ) + hook_module: tuple[str, ...] = ts.option( + factory=list, + help="Path to a Python module that contains hook implementations.", + click={ + "param_decls": ["--hook-module"], + "multiple": True, + "is_eager": True, + "callback": _hook_module_callback, + }, + ) + ignore: tuple[str, ...] = ts.option( + factory=tuple, + help=( + "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " + "for more info." + ), + click={"param_decls": ["--ignore"], "multiple": True}, + ) + paths: tuple[Path, ...] = ts.option( + factory=tuple, + click={ + "param_decls": ["--paths"], + "type": click.Path(exists=True, resolve_path=True, path_type=Path), + "multiple": True, + "callback": _path_callback, + "hidden": True, + }, + ) + pm: PluginManager = ts.option(factory=get_plugin_manager, click={"hidden": True}) + root: Path = ts.option(init=False, click={"hidden": True}) + task_files: tuple[str, ...] = ts.option( + default=("task_*.py",), + help="A list of file patterns for task files.", + click={"param_decls": ["--task-files"], "multiple": True, "hidden": True}, + ) + verbose: int = ts.option( + default=1, + help="Make pytask verbose (>= 0) or quiet (= 0).", + click={ + "param_decls": ["-v", "--verbose"], + "type": click.IntRange(0, 2), + "count": True, + }, + ) + + def __attrs_post_init__(self) -> None: + # Set self.root. + if self.config_file: + self.root = self.config_file.parent + elif self.paths: + candidate = Path(os.path.commonpath(self.paths)) + if candidate.is_dir(): + self.root = candidate + else: + self.root = candidate.parent + else: + self.root = Path.cwd() + + self.cache = self.root / ".pytask" + + +_PATH_ARGUMENT = click.Argument( + ["paths"], + nargs=-1, + type=click.Path(exists=True, resolve_path=True, path_type=Path), ) +"""click.Argument: An argument for paths.""" @hookimpl(trylast=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Register general markers.""" - for command in ("build", "clean", "collect", "dag", "profile"): - cli.commands[command].params.extend((_DATABASE_URL_OPTION,)) - for command in ("build", "clean", "collect", "dag", "markers", "profile"): - cli.commands[command].params.extend( - (_CONFIG_OPTION, _HOOK_MODULE_OPTION, _PATH_ARGUMENT) - ) - for command in ("build", "clean", "collect", "profile"): - cli.commands[command].params.extend([_IGNORE_OPTION, _EDITOR_URL_SCHEME_OPTION]) - for command in ("build",): - cli.commands[command].params.append(_VERBOSE_OPTION) + settings_builder.option_groups["common"] = Common() + settings_builder.arguments.append(_PATH_ARGUMENT) diff --git a/src/_pytask/path.py b/src/_pytask/path.py index aa4dc575..cbc268b9 100644 --- a/src/_pytask/path.py +++ b/src/_pytask/path.py @@ -324,7 +324,7 @@ def shorten_path(path: Path, paths: Sequence[Path]) -> str: when using nested folder structures in bigger projects. Thus, the part of the name which contains the path is replaced by the relative - path from one path in ``session.config["paths"]`` to the node. + path from one path in ``session.config.common.paths`` to the node. """ ancestor = find_closest_ancestor(path, paths) diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 7cb272f0..86e578a4 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from typing import Any from _pytask.dag_utils import node_and_neighbors from _pytask.database_utils import has_node_changed @@ -18,12 +17,13 @@ from _pytask.node_protocols import PTask from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import Settings @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Add the marker to the configuration.""" - config["markers"]["persist"] = ( + config.markers.markers["persist"] = ( "Prevent execution of a task if all products exist and even if something has " "changed (dependencies, source file, products). This decorator might be useful " "for expensive tasks where only the formatting of the file has changed. The " diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index 4674c28b..3159915c 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -46,7 +46,6 @@ def pytask_add_hooks(pm: PluginManager) -> None: "_pytask.dag_command", "_pytask.database", "_pytask.debugging", - "_pytask.provisional", "_pytask.execute", "_pytask.live", "_pytask.logging", diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 71ec5158..893a769c 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -3,22 +3,20 @@ from __future__ import annotations import csv -import enum import json import sys import time from contextlib import suppress +from enum import Enum from typing import TYPE_CHECKING from typing import Any from typing import Generator -import click +import typed_settings as ts from rich.table import Table from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column -from _pytask.click import ColoredCommand -from _pytask.click import EnumChoice from _pytask.console import console from _pytask.console import format_task_name from _pytask.dag import create_dag @@ -31,8 +29,8 @@ from _pytask.outcomes import ExitCode from _pytask.outcomes import TaskOutcome from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings_utils import update_settings from _pytask.traceback import Traceback if TYPE_CHECKING: @@ -40,14 +38,24 @@ from typing import NoReturn from _pytask.reports import ExecutionReport + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder -class _ExportFormats(enum.Enum): +class _ExportFormats(Enum): NO = "no" JSON = "json" CSV = "csv" +@ts.settings +class Profile: + export: _ExportFormats = ts.option( + default=_ExportFormats.NO, + help="Export the profile in the specified format.", + ) + + class Runtime(BaseTable): """Record of runtimes of tasks.""" @@ -59,17 +67,18 @@ class Runtime(BaseTable): @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - cli.add_command(profile) + settings_builder.commands["profile"] = profile_command + settings_builder.option_groups["profile"] = Profile() @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Register the export option.""" - config["pm"].register(ExportNameSpace) - config["pm"].register(DurationNameSpace) - config["pm"].register(FileSizeNameSpace) + config.common.pm.register(ExportNameSpace) + config.common.pm.register(DurationNameSpace) + config.common.pm.register(FileSizeNameSpace) @hookimpl(wrapper=True) @@ -105,21 +114,14 @@ def _create_or_update_runtime(task_signature: str, start: float, end: float) -> session.commit() -@click.command(cls=ColoredCommand) -@click.option( - "--export", - type=EnumChoice(_ExportFormats), - default=_ExportFormats.NO, - help="Export the profile in the specified format.", -) -def profile(**raw_config: Any) -> NoReturn: +def profile_command(settings: Settings, **arguments: Any) -> NoReturn: """Show information about tasks like runtime and memory consumption of products.""" - pm = storage.get() - raw_config["command"] = "profile" + settings = update_settings(settings, arguments) + pm = settings.common.pm try: - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) @@ -158,7 +160,7 @@ def profile(**raw_config: Any) -> NoReturn: def _print_profile_table( - profile: dict[str, dict[str, Any]], tasks: list[PTask], config: dict[str, Any] + profile: dict[str, dict[str, Any]], tasks: list[PTask], config: Settings ) -> None: """Print the profile table.""" name_to_task = {task.name: task for task in tasks} @@ -173,7 +175,7 @@ def _print_profile_table( for task_name, info in profile.items(): task_id = format_task_name( task=name_to_task[task_name], - editor_url_scheme=config["editor_url_scheme"], + editor_url_scheme=config.common.editor_url_scheme, ) infos = [str(i) for i in info.values()] table.add_row(task_id, *infos) @@ -263,12 +265,12 @@ def pytask_profile_export_profile( session: Session, profile: dict[str, dict[str, Any]] ) -> None: """Export profiles.""" - export = session.config["export"] + export = session.config.profile.export if export == _ExportFormats.CSV: - _export_to_csv(profile, session.config["root"]) + _export_to_csv(profile, session.config.common.root) elif export == _ExportFormats.JSON: - _export_to_json(profile, session.config["root"]) + _export_to_json(profile, session.config.common.root) elif export == _ExportFormats.NO: pass else: # pragma: no cover diff --git a/src/_pytask/session.py b/src/_pytask/session.py index 871503b6..e0c32ad2 100644 --- a/src/_pytask/session.py +++ b/src/_pytask/session.py @@ -11,6 +11,7 @@ from pluggy import HookRelay from _pytask.outcomes import ExitCode +from _pytask.settings import Settings if TYPE_CHECKING: from _pytask.node_protocols import PTask @@ -26,6 +27,8 @@ class Session: Parameters ---------- + attrs + A dictionary for storing arbitrary data. config Configuration of the session. collection_reports @@ -49,7 +52,8 @@ class Session: """ - config: dict[str, Any] = field(factory=dict) + attrs: dict[str, Any] = field(factory=dict) + config: Settings = field(factory=Settings) collection_reports: list[CollectionReport] = field(factory=list) dag: nx.DiGraph = field(factory=nx.DiGraph) hook: HookRelay = field(factory=HookRelay) @@ -71,5 +75,7 @@ class Session: @classmethod def from_config(cls, config: dict[str, Any]) -> Session: """Construct the class from a config.""" - hook = config["pm"].hook if "pm" in config else HookRelay() - return cls(config=config, hook=hook) + from _pytask.cli import settings_builder + + settings = settings_builder.load_settings(kwargs=config) + return cls(config=settings, hook=settings.common.pm.hook) diff --git a/src/_pytask/settings.py b/src/_pytask/settings.py new file mode 100644 index 00000000..425d8653 --- /dev/null +++ b/src/_pytask/settings.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import typed_settings as ts + +__all__ = ["Settings"] + + +@ts.settings +class Settings: ... diff --git a/src/_pytask/settings.pyi b/src/_pytask/settings.pyi new file mode 100644 index 00000000..14c76b97 --- /dev/null +++ b/src/_pytask/settings.pyi @@ -0,0 +1,28 @@ +from _pytask.build import Build +from _pytask.capture import Capture +from _pytask.clean import Clean +from _pytask.collect_command import Collect +from _pytask.dag_command import Dag +from _pytask.debugging import Debugging +from _pytask.live import Live +from _pytask.logging import Logging +from _pytask.mark import Markers +from _pytask.parameters import Common +from _pytask.profile import Profile +from _pytask.warnings import Warnings + +__all__ = ["Settings"] + +class Settings: + build: Build + capture: Capture + clean: Clean + collect: Collect + common: Common + dag: Dag + debugging: Debugging + live: Live + logging: Logging + markers: Markers + profile: Profile + warnings: Warnings diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py new file mode 100644 index 00000000..f30299d3 --- /dev/null +++ b/src/_pytask/settings_utils.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import sys +from enum import Enum +from typing import TYPE_CHECKING +from typing import Any +from typing import Callable +from typing import cast + +import attrs +import click +import typed_settings as ts +from attrs import define +from attrs import field +from pluggy import PluginManager +from typed_settings.cli_click import OptionGroupFactory +from typed_settings.exceptions import ConfigFileLoadError +from typed_settings.exceptions import ConfigFileNotFoundError +from typed_settings.types import LoadedSettings +from typed_settings.types import LoaderMeta +from typed_settings.types import OptionList +from typed_settings.types import SettingsClass +from typed_settings.types import SettingsDict + +from _pytask.console import console +from _pytask.settings import Settings + +if TYPE_CHECKING: + from pathlib import Path + + from typed_settings.loaders import Loader + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib # type: ignore[no-redef] + + +__all__ = ["SettingsBuilder", "TomlFormat"] + + +def _handle_enum(type: type[Enum], default: Any, is_optional: bool) -> dict[str, Any]: # noqa: A002 + """Use enum values as choices for click options.""" + kwargs: dict[str, Any] = {"type": click.Choice([i.value for i in type])} + if isinstance(default, type): + kwargs["default"] = default.value + elif is_optional: + kwargs["default"] = None + return kwargs + + +@define +class SettingsBuilder: + commands: dict[str, Callable[..., Any]] = field(factory=dict) + option_groups: dict[str, Any] = field(factory=dict) + arguments: list[Any] = field(factory=list) + + def build_settings(self) -> Any: + return ts.combine("Settings", Settings, self.option_groups) # type: ignore[arg-type] + + def load_settings(self, kwargs: dict[str, Any]) -> Any: + settings = self.build_settings() + return ts.load_settings( + settings, + create_settings_loaders(kwargs=kwargs), + converter=create_converter(), + ) + + def build_decorator(self) -> Any: + settings = self.build_settings() + + type_dict = {**ts.cli_click.DEFAULT_TYPES, Enum: _handle_enum} + type_handler = ts.cli_click.ClickHandler(type_dict) + + return ts.click_options( + settings, + create_settings_loaders(), + converter=create_converter(), + decorator_factory=OptionGroupFactory(), + type_args_maker=ts.cli_utils.TypeArgsMaker(type_handler), + ) + + +_ALREADY_PRINTED_DEPRECATION_MSG: bool = False + + +class TomlFormat: + """Support for TOML files.""" + + def __init__( + self, + section: str | None, + exclude: list[str] | None = None, + deprecated: str = "", + ) -> None: + self.section = section + self.exclude = exclude or [] + self.deprecated = deprecated + + def __call__( + self, + path: Path, + settings_cls: SettingsClass, # noqa: ARG002 + options: OptionList, + ) -> SettingsDict: + """Load settings from a TOML file and return them as a dict.""" + try: + with path.open("rb") as f: + settings = tomllib.load(f) + except FileNotFoundError as e: + raise ConfigFileNotFoundError(str(e)) from e + except (PermissionError, tomllib.TOMLDecodeError) as e: + raise ConfigFileLoadError(str(e)) from e + if self.section is not None: + sections = self.section.split(".") + for s in sections: + try: + settings = settings[s] + except KeyError: # noqa: PERF203 + return {} + for key in self.exclude: + settings.pop(key, None) + + global _ALREADY_PRINTED_DEPRECATION_MSG # noqa: PLW0603 + if self.deprecated and not _ALREADY_PRINTED_DEPRECATION_MSG: + _ALREADY_PRINTED_DEPRECATION_MSG = True + console.print(self.deprecated, style="skipped") + settings["common.config_file"] = path + settings["common.root"] = path.parent + settings = _rewrite_paths_of_options(settings, options, section=self.section) + return cast(SettingsDict, settings) + + +class DictLoader: + """Load settings from a dict of values.""" + + def __init__(self, settings: dict[str, Any]) -> None: + self.settings = settings + + def __call__( + self, + settings_cls: SettingsClass, # noqa: ARG002 + options: OptionList, + ) -> LoadedSettings: + settings = _rewrite_paths_of_options(self.settings, options, section=None) + nested_settings: dict[str, dict[str, Any]] = { + name.split(".")[0]: {} for name in settings + } + for long_name, value in settings.items(): + group, name = long_name.split(".") + nested_settings[group][name] = value + return LoadedSettings(nested_settings, LoaderMeta(self)) + + +def load_settings(settings_cls: Any, kwargs: dict[str, Any] | None = None) -> Any: + """Load the settings.""" + loaders = create_settings_loaders(kwargs=kwargs) + converter = create_converter() + return ts.load_settings(settings_cls, loaders, converter=converter) + + +def _convert_to_enum(val: Any, cls: type[Enum]) -> Enum: + if isinstance(val, Enum): + return val + try: + return cls(val) + except ValueError: + values = ", ".join([i.value for i in cls]) + msg = ( + f"{val!r} is not a valid value for {cls.__name__}. Use one of {values} " + "instead." + ) + raise ValueError(msg) from None + + +def create_converter() -> ts.Converter: + """Create the converter.""" + converter = ts.converters.get_default_ts_converter() + converter.scalar_converters[Enum] = _convert_to_enum + converter.scalar_converters[PluginManager] = ( + lambda val, cls: val if isinstance(val, cls) else cls(**val) + ) + return converter + + +def create_settings_loaders(kwargs: dict[str, Any] | None = None) -> list[Loader]: + """Create the loaders for the settings.""" + kwargs_ = kwargs or {} + return [ + ts.FileLoader( + files=[ts.find("pyproject.toml")], + env_var=None, + formats={ + "*.toml": TomlFormat( + section="tool.pytask.ini_options", + deprecated=( + "DeprecationWarning: Configuring pytask in the " + "section \\[tool.pytask.ini_options] is deprecated and will be " + "removed in v0.6. Please, use \\[tool.pytask] instead." + ), + ) + }, + ), + ts.FileLoader( + files=[ts.find("pyproject.toml")], + env_var=None, + formats={ + "*.toml": TomlFormat(section="tool.pytask", exclude=["ini_options"]) + }, + ), + ts.EnvLoader(prefix="PYTASK_", nested_delimiter="_"), + DictLoader(kwargs_), + ] + + +def update_settings(settings: Any, updates: dict[str, Any]) -> Any: + """Update the settings recursively with some updates.""" + names = [i for i in dir(settings) if not i.startswith("_")] + for name in names: + if attrs.has(getattr(settings, name)): + update_settings(getattr(settings, name), updates) + continue + + if name in updates: + value = updates[name] + if value in ((), []): + continue + setattr(settings, name, updates[name]) + return settings + + +def convert_settings_to_kwargs(settings: Settings) -> dict[str, Any]: + """Convert the settings to kwargs.""" + kwargs: dict[str, Any] = {} + names = [i for i in dir(settings) if not i.startswith("_")] + for name in names: + kwargs = kwargs | attrs.asdict(getattr(settings, name)) + return kwargs + + +def _rewrite_paths_of_options( + settings: SettingsDict, options: OptionList, section: str | None +) -> SettingsDict: + """Rewrite paths of options in the settings.""" + option_paths = {option.path for option in options} + option_name_to_path = { + option.path.rsplit(".", maxsplit=1)[1]: option.path for option in options + } + + new_settings = {} + for name, value in settings.items(): + if name in option_paths: + new_settings[name] = value + continue + + if name in option_name_to_path: + new_path = option_name_to_path[name] + if section: + subsection, _ = new_path.rsplit(".", maxsplit=1) + msg = ( + f"DeprecationWarning: The path of the option {name!r} changed from " + f"\\[{section}] to the new path \\[tool.pytask.{subsection}]." + ) + console.print(msg, style="skipped") + new_settings[new_path] = value + + return new_settings diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index b7a2f491..86928ec8 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -63,19 +63,18 @@ def to_list(scalar_or_iter: Any) -> list[Any]: ) -def parse_paths(x: Path | list[Path]) -> list[Path]: +def parse_paths(x: tuple[Path, ...]) -> tuple[Path, ...]: """Parse paths.""" paths = [Path(p) for p in to_list(x)] for p in paths: if not p.exists(): msg = f"The path '{p}' does not exist." raise FileNotFoundError(msg) - - return [ + return tuple( Path(p).resolve() for path in paths for p in glob.glob(path.as_posix()) # noqa: PTH207 - ] + ) def reduce_names_of_multiple_nodes( diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index a7678154..39ebc49c 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from typing import Any from _pytask.dag_utils import descending_tasks from _pytask.mark import Mark @@ -20,6 +19,7 @@ from _pytask.node_protocols import PTask from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import Settings def skip_ancestor_failed(reason: str = "No reason provided.") -> str: @@ -33,7 +33,7 @@ def skipif(condition: bool, *, reason: str) -> tuple[bool, str]: @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" markers = { "skip": "Skip a task and all its dependent tasks.", @@ -43,7 +43,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: "executed and have not been changed.", "skipif": "Skip a task and all its dependent tasks if a condition is met.", } - config["markers"] = {**config["markers"], **markers} + config.markers.markers = {**config.markers.markers, **markers} @hookimpl @@ -52,7 +52,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: is_unchanged = has_mark(task, "skip_unchanged") and not has_mark( task, "would_be_executed" ) - if is_unchanged and not session.config["force"]: + if is_unchanged and not session.config.build.force: collect_provisional_products(session, task) raise SkippedUnchanged diff --git a/src/_pytask/task.py b/src/_pytask/task.py index 902f6eb0..831dd259 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -17,12 +17,13 @@ from _pytask.reports import CollectionReport from _pytask.session import Session + from _pytask.settings import Settings @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" - config["markers"]["task"] = ( + config.markers.markers["task"] = ( "Mark a function as a task regardless of its name. Or mark tasks which are " "repeated in a loop. See this tutorial for more information: " "[link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/]." @@ -35,7 +36,7 @@ def pytask_collect_file( ) -> list[CollectionReport] | None: """Collect a file.""" if ( - any(path.match(pattern) for pattern in session.config["task_files"]) + any(path.match(pattern) for pattern in session.config.common.task_files) and COLLECTED_TASKS[path] ): # Remove tasks from the global to avoid re-collection if programmatic interface diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index 7ee00178..e3bf37b0 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -24,7 +24,6 @@ if TYPE_CHECKING: from pathlib import Path - __all__ = [ "COLLECTED_TASKS", "parse_collected_tasks_with_task_marker", diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index 00242916..681aed1e 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -12,6 +12,7 @@ from typing import Union import pluggy +import typed_settings as ts from attrs import define from attrs import field from rich.traceback import Traceback as RichTraceback @@ -35,6 +36,14 @@ _PLUGGY_DIRECTORY = Path(pluggy.__file__).parent _PYTASK_DIRECTORY = Path(_pytask.__file__).parent +_TYPED_SETTINGS_DIRECTORY = Path(ts.__file__).parent + +_DEFAULT_SUPPRESS = ( + _PLUGGY_DIRECTORY, + _PYTASK_DIRECTORY, + _TYPED_SETTINGS_DIRECTORY, + TREE_UTIL_LIB_DIRECTORY, +) ExceptionInfo: TypeAlias = Tuple[ @@ -49,11 +58,7 @@ class Traceback: show_locals: bool = field() _show_locals: ClassVar[bool] = False - suppress: ClassVar[tuple[Path, ...]] = ( - _PLUGGY_DIRECTORY, - _PYTASK_DIRECTORY, - TREE_UTIL_LIB_DIRECTORY, - ) + suppress: ClassVar[tuple[Path, ...]] = _DEFAULT_SUPPRESS @show_locals.default def _show_locals_default(self) -> bool: @@ -90,6 +95,7 @@ def _remove_internal_traceback_frames_from_exc_info( suppress: tuple[Path, ...] = ( _PLUGGY_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, + _TYPED_SETTINGS_DIRECTORY, _PYTASK_DIRECTORY, ), ) -> OptionalExceptionInfo: @@ -103,6 +109,9 @@ def _remove_internal_traceback_frames_from_exc_info( exc_info[1].__cause__ = _remove_internal_traceback_frames_from_exception( exc_info[1].__cause__ ) + exc_info[1].__context__ = _remove_internal_traceback_frames_from_exception( + exc_info[1].__context__ + ) if isinstance(exc_info[2], TracebackType): filtered_traceback = _filter_internal_traceback_frames(exc_info, suppress) @@ -132,11 +141,7 @@ def _remove_internal_traceback_frames_from_exception( def _is_internal_or_hidden_traceback_frame( frame: TracebackType, exc_info: ExceptionInfo, - suppress: tuple[Path, ...] = ( - _PLUGGY_DIRECTORY, - TREE_UTIL_LIB_DIRECTORY, - _PYTASK_DIRECTORY, - ), + suppress: tuple[Path, ...] = _DEFAULT_SUPPRESS, ) -> bool: """Return ``True`` if traceback frame belongs to internal packages or is hidden. diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index b5986c84..eb2040cc 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -4,10 +4,9 @@ from collections import defaultdict from typing import TYPE_CHECKING -from typing import Any from typing import Generator -import click +import typed_settings as ts from attrs import define from rich.padding import Padding from rich.panel import Panel @@ -16,7 +15,6 @@ from _pytask.pluginmanager import hookimpl from _pytask.warnings_utils import WarningReport from _pytask.warnings_utils import catch_warnings_for_item -from _pytask.warnings_utils import parse_filterwarnings if TYPE_CHECKING: from rich.console import Console @@ -25,35 +23,43 @@ from _pytask.node_protocols import PTask from _pytask.session import Session + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + + +@ts.settings +class Warnings: + """Settings for warnings.""" + + filterwarnings: list[str] = ts.option( + factory=list, + click={"hidden": True}, + help="Add a filter for a warning to a task.", + ) + disable_warnings: bool = ts.option( + default=False, + click={"param_decls": ["--disable-warnings"], "is_flag": True}, + help="Disables the summary for warnings.", + ) @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the cli.""" - cli.commands["build"].params.extend( - [ - click.Option( - ["--disable-warnings"], - is_flag=True, - default=False, - help="Disables the summary for warnings.", - ) - ] - ) + settings_builder.option_groups["warnings"] = Warnings() @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" - config["filterwarnings"] = parse_filterwarnings(config.get("filterwarnings")) - config["markers"]["filterwarnings"] = "Add a filter for a warning to a task." + config.markers.markers["filterwarnings"] = "Add a filter for a warning to a task." @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Activate the warnings plugin if not disabled.""" - if not config["disable_warnings"]: - config["pm"].register(WarningsNameSpace) + if not config.warnings.disable_warnings: + config.common.pm.register(WarningsNameSpace) class WarningsNameSpace: diff --git a/src/_pytask/warnings_utils.py b/src/_pytask/warnings_utils.py index 61c6597b..7dacd985 100644 --- a/src/_pytask/warnings_utils.py +++ b/src/_pytask/warnings_utils.py @@ -167,7 +167,7 @@ def catch_warnings_for_item( # mypy can't infer that record=True means log is not None; help it. assert log is not None - for arg in session.config["filterwarnings"]: + for arg in session.config.warnings.filterwarnings: warnings.filterwarnings(*parse_warning_filter(arg, escape=False)) # apply filters from "filterwarnings" marks diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 4caec161..542a8fbb 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -12,7 +12,6 @@ from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup -from _pytask.click import EnumChoice from _pytask.collect_utils import parse_dependencies_from_task_function from _pytask.collect_utils import parse_products_from_task_function from _pytask.compat import check_for_optional_program @@ -98,7 +97,6 @@ "DataCatalog", "DatabaseSession", "DirectoryNode", - "EnumChoice", "ExecutionError", "ExecutionReport", "Exit", diff --git a/tests/test_capture.py b/tests/test_capture.py index a157307a..c4d5a879 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -561,9 +561,8 @@ def test_simple_resume_suspend(self): pytest.raises(AssertionError, cap.suspend) assert repr(cap) == ( - "".format( # noqa: UP032 - cap.targetfd_save, cap.tmpfile - ) + f"" ) # Should not crash with missing "_old". assert repr(cap.syscapture) == ( diff --git a/tests/test_cli.py b/tests/test_cli.py index 01b40c84..66af9043 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -34,14 +34,21 @@ def test_help_pages(runner, commands, help_option): @pytest.mark.end_to_end() -def test_help_texts_are_modified_by_config(runner, tmp_path): +@pytest.mark.parametrize("config_section", ["pytask.ini_options", "pytask"]) +def test_help_texts_are_modified_by_config(tmp_path, config_section): tmp_path.joinpath("pyproject.toml").write_text( - '[tool.pytask.ini_options]\nshow_capture = "stdout"' + f'[tool.{config_section}]\nshow_capture = "stdout"' ) + result = run_in_subprocess(("pytask", "build", "--help"), cwd=tmp_path) + assert "[default:" in result.stdout + assert " stdout]" in result.stdout - result = runner.invoke( - cli, - ["build", "--help", "--config", tmp_path.joinpath("pyproject.toml").as_posix()], - ) - assert "[default: stdout]" in result.output +def test_precendence_of_new_to_old_section(tmp_path): + tmp_path.joinpath("pyproject.toml").write_text( + '[tool.pytask.ini_options]\nshow_capture = "stdout"\n\n' + '[tool.pytask]\nshow_capture = "stderr"' + ) + result = run_in_subprocess(("pytask", "build", "--help"), cwd=tmp_path) + assert "[default:" in result.stdout + assert " stderr]" in result.stdout diff --git a/tests/test_click.py b/tests/test_click.py index ec01e758..f6ee1a79 100644 --- a/tests/test_click.py +++ b/tests/test_click.py @@ -1,10 +1,6 @@ from __future__ import annotations -import enum - -import click import pytest -from pytask import EnumChoice from pytask import cli @@ -19,38 +15,3 @@ def test_choices_are_displayed_in_help_page(runner): def test_defaults_are_displayed(runner): result = runner.invoke(cli, ["build", "--help"]) assert "[default: all]" in result.output - - -@pytest.mark.unit() -@pytest.mark.parametrize("method", ["first", "second"]) -def test_enum_choice(runner, method): - class Method(enum.Enum): - FIRST = "first" - SECOND = "second" - - @click.command() - @click.option("--method", type=EnumChoice(Method)) - def test(method): - print(f"method={method}") # noqa: T201 - - result = runner.invoke(test, ["--method", method]) - - assert result.exit_code == 0 - assert f"method=Method.{method.upper()}" in result.output - - -@pytest.mark.unit() -def test_enum_choice_error(runner): - class Method(enum.Enum): - FIRST = "first" - SECOND = "second" - - @click.command() - @click.option("--method", type=EnumChoice(Method)) - def test(): ... - - result = runner.invoke(test, ["--method", "third"]) - - assert result.exit_code == 2 - assert "Invalid value for '--method': " in result.output - assert "'third' is not one of 'first', 'second'." in result.output diff --git a/tests/test_collect.py b/tests/test_collect.py index 93591998..d10c4966 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -16,6 +16,8 @@ from pytask import build from pytask import cli +from tests.conftest import enter_directory + @pytest.mark.end_to_end() @pytest.mark.parametrize( @@ -111,7 +113,7 @@ def test_collect_same_task_different_ways(tmp_path, path_extension): ], ) def test_collect_files_w_custom_file_name_pattern( - tmp_path, task_files, pattern, expected_collected_tasks + runner, tmp_path, task_files, pattern, expected_collected_tasks ): tmp_path.joinpath("pyproject.toml").write_text( f"[tool.pytask.ini_options]\ntask_files = {pattern}" @@ -120,10 +122,10 @@ def test_collect_files_w_custom_file_name_pattern( for file_ in task_files: tmp_path.joinpath(file_).write_text("def task_example(): pass") - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.OK - assert len(session.tasks) == expected_collected_tasks + with enter_directory(tmp_path): + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert f"Collected {expected_collected_tasks} task" in result.output def test_error_with_invalid_file_name_pattern(runner, tmp_path): diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index aabe7036..ab5dbf5d 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -1,10 +1,14 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest from _pytask.collect_utils import _find_args_with_product_annotation -from pytask import Product # noqa: TCH002 from typing_extensions import Annotated +if TYPE_CHECKING: + from pytask import Product + @pytest.mark.unit() def test_find_args_with_product_annotation(): diff --git a/tests/test_config.py b/tests/test_config.py index 874bdf62..92484e6d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -39,7 +39,7 @@ def test_pass_config_to_cli(tmp_path): session = build(config=tmp_path.joinpath("pyproject.toml"), paths=tmp_path) assert session.exit_code == ExitCode.OK - assert "elton" in session.config["markers"] + assert "elton" in session.config.markers.markers @pytest.mark.end_to_end() @@ -123,3 +123,15 @@ def test_paths_are_relative_to_configuration_file(tmp_path): result = run_in_subprocess(("python", "script.py"), cwd=tmp_path) assert result.exit_code == ExitCode.OK assert "1 Succeeded" in result.stdout + + +def test_old_config_section_is_deprecated(): ... + + +def test_new_config_section_is_not_deprecated(): ... + + +def test_old_config_path_is_deprecated(): ... + + +def test_new_config_path_is_not_deprecated(): ... diff --git a/tests/test_database.py b/tests/test_database.py index 0745a999..cd1f7898 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -7,7 +7,6 @@ from pytask import ExitCode from pytask import State from pytask import build -from pytask import cli from pytask import create_database from pytask.path import hash_path from sqlalchemy.engine import make_url @@ -50,27 +49,3 @@ def task_write(path=Path("in.txt"), produces=Path("out.txt")): ): hash_ = db_session.get(State, (task_id, id_)).hash_ assert hash_ == hash_path(path, path.stat().st_mtime) - - -@pytest.mark.end_to_end() -def test_rename_database_w_config(tmp_path, runner): - """Modification dates of input and output files are stored in database.""" - path_to_db = tmp_path.joinpath(".db.sqlite") - tmp_path.joinpath("pyproject.toml").write_text( - "[tool.pytask.ini_options]\ndatabase_url='sqlite:///.db.sqlite'" - ) - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert path_to_db.exists() - - -@pytest.mark.end_to_end() -def test_rename_database_w_cli(tmp_path, runner): - """Modification dates of input and output files are stored in database.""" - path_to_db = tmp_path.joinpath(".db.sqlite") - result = runner.invoke( - cli, - ["--database-url", "sqlite:///.db.sqlite", tmp_path.as_posix()], - ) - assert result.exit_code == ExitCode.OK - assert path_to_db.exists() diff --git a/tests/test_jupyter/test_functional_interface.ipynb b/tests/test_jupyter/test_functional_interface.ipynb index 3abd995c..c74eb5e2 100644 --- a/tests/test_jupyter/test_functional_interface.ipynb +++ b/tests/test_jupyter/test_functional_interface.ipynb @@ -9,10 +9,11 @@ "source": [ "from pathlib import Path\n", "\n", - "from typing_extensions import Annotated\n", - "\n", "import pytask\n", - "from pytask import ExitCode, PathNode, PythonNode" + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" ] }, { diff --git a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb index fe0125df..5226c122 100644 --- a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb +++ b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb @@ -9,10 +9,11 @@ "source": [ "from pathlib import Path\n", "\n", - "from typing_extensions import Annotated\n", - "\n", "import pytask\n", - "from pytask import ExitCode, PathNode, PythonNode" + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" ] }, { diff --git a/tests/test_jupyter/test_task_generator.ipynb b/tests/test_jupyter/test_task_generator.ipynb index 2ef6aa61..d6d4c4b8 100644 --- a/tests/test_jupyter/test_task_generator.ipynb +++ b/tests/test_jupyter/test_task_generator.ipynb @@ -11,10 +11,11 @@ "\n", "from pathlib import Path\n", "\n", - "from typing_extensions import Annotated\n", - "\n", "import pytask\n", - "from pytask import DirectoryNode, ExitCode, task" + "from pytask import DirectoryNode\n", + "from pytask import ExitCode\n", + "from pytask import task\n", + "from typing_extensions import Annotated" ] }, { @@ -32,7 +33,7 @@ "\n", "@task(after=task_create_files, is_generator=True)\n", "def task_generator_copy_files(\n", - " paths: Annotated[list[Path], DirectoryNode(pattern=\"[ab].txt\")]\n", + " paths: Annotated[list[Path], DirectoryNode(pattern=\"[ab].txt\")],\n", "):\n", " for path in paths:\n", "\n", diff --git a/tests/test_persist.py b/tests/test_persist.py index 46580fb4..bd609be5 100644 --- a/tests/test_persist.py +++ b/tests/test_persist.py @@ -24,7 +24,7 @@ class DummyClass: @pytest.mark.end_to_end() def test_persist_marker_is_set(tmp_path): session = build(paths=tmp_path) - assert "persist" in session.config["markers"] + assert "persist" in session.config.markers.markers @pytest.mark.end_to_end() diff --git a/tests/test_shared.py b/tests/test_shared.py index 882daba5..ef5e35c1 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -36,8 +36,8 @@ def test_parse_markers(tmp_path): session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK - assert "a1" in session.config["markers"] - assert "a2" in session.config["markers"] + assert "a1" in session.config.markers.markers + assert "a2" in session.config.markers.markers @pytest.mark.end_to_end() diff --git a/tox.ini b/tox.ini index 22051f4f..096c6f7a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -envlist = docs, test +envlist = docs, mypy, test [testenv] passenv = CI