diff --git a/README.md b/README.md index 8112672..70b8ccd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Styx definitions and minimal runtime +# Styx type definitions and minimal runtime [![Build](https://github.com/childmindresearch/styxdefs/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/childmindresearch/styxdefs/actions/workflows/test.yaml?query=branch%3Amain) [![codecov](https://codecov.io/gh/childmindresearch/styxdefs/branch/main/graph/badge.svg?token=22HWWFWPW5)](https://codecov.io/gh/childmindresearch/styxdefs) @@ -6,3 +6,12 @@ ![stability-stable](https://img.shields.io/badge/stability-stable-green.svg) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/childmindresearch/styxdefs/blob/main/LICENSE) [![pages](https://img.shields.io/badge/api-docs-blue)](https://childmindresearch.github.io/styxdefs) + +Type definitions and minimal runtime for [Styx](https://github.com/childmindresearch/styx) generated wrappers and +Styx Runners. This package defines the common API for all Styx generated code. + +## Installation + +```bash +pip install styxdefs +``` diff --git a/poetry.lock b/poetry.lock index 8157d51..aaea106 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "cfgv" @@ -102,13 +102,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.1" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [package.extras] diff --git a/pyproject.toml b/pyproject.toml index 0cb575a..b712697 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,43 +32,19 @@ testpaths = [ ignore_missing_imports = true [tool.ruff] -exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".mypy_cache", - ".nox", - ".pants.d", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "venv" -] line-length = 88 indent-width = 4 src = ["src"] target-version = "py311" [tool.ruff.lint] -select = ["ANN", "D", "E", "F", "I"] +select = ["ANN", "D", "E", "F", "I", "UP"] ignore = [ "ANN101", # self should not be annotated. "ANN102" # cls should not be annotated. ] fixable = ["ALL"] unfixable = [] -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" [tool.ruff.lint.pydocstyle] convention = "google" @@ -76,12 +52,6 @@ convention = "google" [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = [] -[tool.ruff.format] -quote-style = "double" -indent-style = "space" -skip-magic-trailing-comma = false -line-ending = "auto" - [build-system] requires = ["poetry-core>=1.2.0"] build-backend = "poetry.core.masonry.api" diff --git a/src/styxdefs/__init__.py b/src/styxdefs/__init__.py index 1db1f29..18643bf 100644 --- a/src/styxdefs/__init__.py +++ b/src/styxdefs/__init__.py @@ -1,16 +1,19 @@ """.. include:: ../../README.md""" # noqa: D415 -from .runner import ( - DefaultRunner, +from .global_state import ( get_global_runner, set_global_runner, ) +from .local_runner import ( + LocalRunner, +) from .types import ( Execution, InputPathType, Metadata, OutputPathType, Runner, + StyxRuntimeError, ) __all__ = [ @@ -19,7 +22,8 @@ "Metadata", "OutputPathType", "Runner", - "DefaultRunner", + "LocalRunner", "get_global_runner", "set_global_runner", + "StyxRuntimeError", ] diff --git a/src/styxdefs/global_state.py b/src/styxdefs/global_state.py new file mode 100644 index 0000000..fdc8648 --- /dev/null +++ b/src/styxdefs/global_state.py @@ -0,0 +1,20 @@ +"""Global state for the Styx library.""" + +from .local_runner import LocalRunner +from .types import Runner + +_STYX_GLOBAL_RUNNER: Runner | None = None + + +def get_global_runner() -> Runner: + """Get the default runner.""" + global _STYX_GLOBAL_RUNNER + if _STYX_GLOBAL_RUNNER is None: + _STYX_GLOBAL_RUNNER = LocalRunner() + return _STYX_GLOBAL_RUNNER + + +def set_global_runner(runner: Runner) -> None: + """Set the default runner.""" + global _STYX_GLOBAL_RUNNER + _STYX_GLOBAL_RUNNER = runner diff --git a/src/styxdefs/runner.py b/src/styxdefs/local_runner.py similarity index 56% rename from src/styxdefs/runner.py rename to src/styxdefs/local_runner.py index 5c2de1d..cb35b88 100644 --- a/src/styxdefs/runner.py +++ b/src/styxdefs/local_runner.py @@ -1,30 +1,45 @@ -"""Default runner implementation.""" +"""Simple local runner implementation.""" import logging import os import pathlib +import shlex from collections import deque from concurrent.futures import ThreadPoolExecutor +from datetime import datetime from functools import partial -from subprocess import PIPE, CalledProcessError, Popen - -from .types import Execution, InputPathType, Metadata, OutputPathType, Runner - - -class _DefaultExecution(Execution): - """Default execution implementation.""" - - def __init__(self, logger: logging.Logger, dir: pathlib.Path) -> None: +from subprocess import PIPE, Popen + +from .types import ( + Execution, + InputPathType, + Metadata, + OutputPathType, + Runner, + StyxRuntimeError, +) + + +class _LocalExecution(Execution): + """Local execution object.""" + + def __init__( + self, + logger: logging.Logger, + output_dir: pathlib.Path, + metadata: Metadata, + ) -> None: """Initialize the execution.""" self.logger: logging.Logger = logger - self.dir: pathlib.Path = dir + self.output_dir: pathlib.Path = output_dir + self.metadata: Metadata = metadata - while self.dir.exists(): + while self.output_dir.exists(): self.logger.warning( - f"Execution directory {self.dir} already exists. Trying another." + f"Output directory {self.output_dir} already exists. Trying another." ) - self.dir = self.dir.with_name(f"{self.dir.name}_1") - self.dir.mkdir(parents=True, exist_ok=True) + self.output_dir = self.output_dir.with_name(f"{self.output_dir.name}_1") + self.output_dir.mkdir(parents=True, exist_ok=True) def input_file(self, host_file: InputPathType) -> str: """Resolve host input files.""" @@ -32,11 +47,11 @@ def input_file(self, host_file: InputPathType) -> str: def output_file(self, local_file: str, optional: bool = False) -> OutputPathType: """Resolve local output files.""" - return self.dir / local_file + return self.output_dir / local_file def run(self, cargs: list[str]) -> None: """Run the command.""" - self.logger.debug(f"Running command: {cargs} in '{self.dir}'.") + self.logger.debug(f"Running command: {shlex.join(cargs)}") def _stdout_handler(line: str) -> None: self.logger.info(line) @@ -44,20 +59,25 @@ def _stdout_handler(line: str) -> None: def _stderr_handler(line: str) -> None: self.logger.error(line) - with Popen(cargs, text=True, stdout=PIPE, stderr=PIPE, cwd=self.dir) as process: + time_start = datetime.now() + with Popen( + cargs, text=True, stdout=PIPE, stderr=PIPE, cwd=self.output_dir + ) as process: with ThreadPoolExecutor(2) as pool: # two threads to handle the streams exhaust = partial(pool.submit, partial(deque, maxlen=0)) exhaust(_stdout_handler(line[:-1]) for line in process.stdout) # type: ignore exhaust(_stderr_handler(line[:-1]) for line in process.stderr) # type: ignore return_code = process.poll() + time_end = datetime.now() + self.logger.info(f"Executed {self.metadata.name} in {time_end - time_start}") if return_code: - raise CalledProcessError(return_code, process.args) + raise StyxRuntimeError(return_code, cargs) -class DefaultRunner(Runner): - """Default runner implementation.""" +class LocalRunner(Runner): + """Local runner implementation.""" - logger_name = "styx_default_runner" + logger_name = "styx_local_runner" def __init__(self, data_dir: InputPathType | None = None) -> None: """Initialize the runner.""" @@ -77,26 +97,12 @@ def __init__(self, data_dir: InputPathType | None = None) -> None: def start_execution(self, metadata: Metadata) -> Execution: """Start a new execution.""" + output_dir = ( + self.data_dir / f"{self.uid}_{self.execution_counter}_{metadata.name}" + ) self.execution_counter += 1 - return _DefaultExecution( + return _LocalExecution( logger=self.logger, - dir=self.data_dir - / f"{self.uid}_{self.execution_counter - 1}_{metadata.name}", + output_dir=output_dir, + metadata=metadata, ) - - -_DEFAULT_RUNNER: Runner | None = None - - -def get_global_runner() -> Runner: - """Get the default runner.""" - global _DEFAULT_RUNNER - if _DEFAULT_RUNNER is None: - _DEFAULT_RUNNER = DefaultRunner() - return _DEFAULT_RUNNER - - -def set_global_runner(runner: Runner) -> None: - """Set the default runner.""" - global _DEFAULT_RUNNER - _DEFAULT_RUNNER = runner diff --git a/src/styxdefs/types.py b/src/styxdefs/types.py index ba35ea1..80aa470 100644 --- a/src/styxdefs/types.py +++ b/src/styxdefs/types.py @@ -1,6 +1,7 @@ """Types for styx generated wrappers.""" import pathlib +import shlex import typing InputPathType: typing.TypeAlias = pathlib.Path | str @@ -53,6 +54,10 @@ class Metadata(typing.NamedTuple): """Unique identifier of the tool.""" name: str """Name of the tool.""" + package: str | None = None + """Name of the package that provides the tool.""" + citations: list[str] | None = None + """List of references to cite when using the tool.""" container_image_type: str | None = None """Type of container image. Example: docker, singularity.""" container_image_tag: str | None = None @@ -83,3 +88,29 @@ def start_execution(self, metadata: Metadata) -> Execution: Called before any `Execution.input_file()` calls. """ ... + + +class StyxRuntimeError(Exception): + """Styx runtime error. + + Raised when a command reports a non-zero return code. + """ + + def __init__( + self, + return_code: int | None = None, + command_args: list[str] | None = None, + ) -> None: + """Initialize the error.""" + self.return_code = return_code + self.command_args = command_args + + message = "Command failed." + + if return_code is not None: + message += f"\n- Return code: {return_code}" + + if command_args is not None: + message += f"\n- Command args: {shlex.join(command_args)}" + + super().__init__(message) diff --git a/tests/test_dummy.py b/tests/test_dummy.py deleted file mode 100644 index 208542e..0000000 --- a/tests/test_dummy.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Dummy test.""" - - -def test_dummy() -> None: - """Test dummy.""" - assert True diff --git a/tests/test_global_state.py b/tests/test_global_state.py new file mode 100644 index 0000000..4a5d830 --- /dev/null +++ b/tests/test_global_state.py @@ -0,0 +1,14 @@ +"""Test global state.""" + +import styxdefs + + +def test_global_runner() -> None: + """Test the global runner.""" + runner = styxdefs.get_global_runner() + assert hasattr(runner, "start_execution") + assert isinstance(runner, styxdefs.LocalRunner) + + styxdefs.set_global_runner(styxdefs.LocalRunner(data_dir="xyz")) + runner = styxdefs.get_global_runner() + assert isinstance(runner, styxdefs.LocalRunner) diff --git a/tests/test_local_runner.py b/tests/test_local_runner.py new file mode 100644 index 0000000..f8b6c2d --- /dev/null +++ b/tests/test_local_runner.py @@ -0,0 +1,27 @@ +"""Test the local runner.""" + +import os +import pathlib + +import styxdefs + + +def test_local_runner(tmp_path: pathlib.Path) -> None: + """Test the local runner.""" + runner = styxdefs.LocalRunner(data_dir=tmp_path / "xyz") + + x = runner.start_execution( + styxdefs.Metadata( + id="123", + name="test", + ) + ) + + input_file = x.input_file("abc") + output_file = x.output_file("def") + if os.name == "posix": + x.run(["ls"]) + + assert pathlib.Path(input_file).name == "abc" + assert output_file.is_relative_to(tmp_path / "xyz") + assert output_file.name == "def" diff --git a/tests/test_types.py b/tests/test_types.py new file mode 100644 index 0000000..c5acdd4 --- /dev/null +++ b/tests/test_types.py @@ -0,0 +1,12 @@ +"""Tests for the types module.""" + +import styxdefs.types + + +def test_runtime_error() -> None: + """Test the StyxRuntimeError class.""" + try: + raise styxdefs.types.StyxRuntimeError(1, ["ls", "-l"]) + except styxdefs.types.StyxRuntimeError as e: + assert e.return_code == 1 + assert e.command_args == ["ls", "-l"]