diff --git a/docs/conf.py b/docs/conf.py index b3f622297..c5f5685d0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,18 +24,28 @@ "sphinx.ext.viewcode", "sphinx.ext.napoleon", "myst_parser", + "autoapi.extension", ] master_doc = "index" +autoapi_type = "python" +autoapi_dirs = ["../src/hera"] # Starts from the `hera` module, without including `src` in the name +autoapi_file_pattern = "*.py" +autoapi_python_use_implicit_namespaces = True +autoapi_add_toctree_entry = False + templates_path = ["_templates"] -exclude_patterns = [ - "_build", - "Thumbs.db", - ".DS_Store", - "conftest.py", - "tests", - "\.github", -] +exclude_patterns = ( + [ # uses glob-style https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-exclude_patterns + "_[!_]*[!_][!_].py", # exclude private modules like `_mixins.py` but not dunder files (not working?) + "_build", + "Thumbs.db", + ".DS_Store", + "conftest.py", + "tests", + ".github", + ] +) numpydoc_show_inherited_class_members = True numpydoc_show_class_members = False diff --git a/docs/examples/workflows/upstream/cluster_workflow_template__workflow_template_ref.md b/docs/examples/workflows/upstream/cluster_workflow_template__workflow_template_ref.md new file mode 100644 index 000000000..d055436d3 --- /dev/null +++ b/docs/examples/workflows/upstream/cluster_workflow_template__workflow_template_ref.md @@ -0,0 +1,32 @@ +# Cluster Workflow Template Workflow Template Ref + +> Note: This example is a replication of an Argo Workflow example in Hera. The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/master/examples/cluster-workflow-template/workflow-template-ref.yaml). + + + +## Hera + +```python +from hera.workflows import Workflow +from hera.workflows.models import WorkflowTemplateRef + +wt_ref = WorkflowTemplateRef(name="cluster-workflow-template-submittable", cluster_scope=True) + +w = Workflow( + generate_name="cluster-workflow-template-hello-world-", + workflow_template_ref=wt_ref, +) +``` + +## YAML + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: cluster-workflow-template-hello-world- +spec: + workflowTemplateRef: + clusterScope: true + name: cluster-workflow-template-submittable +``` diff --git a/docs/examples/workflows/upstream/workflow_template__workflow_template_ref.md b/docs/examples/workflows/upstream/workflow_template__workflow_template_ref.md index 6ddcb7b42..cf72bce1f 100644 --- a/docs/examples/workflows/upstream/workflow_template__workflow_template_ref.md +++ b/docs/examples/workflows/upstream/workflow_template__workflow_template_ref.md @@ -12,11 +12,10 @@ from hera.workflows.models import WorkflowTemplateRef wt_ref = WorkflowTemplateRef(name="workflow-template-submittable") -with Workflow( +w = Workflow( generate_name="workflow-template-hello-world-", workflow_template_ref=wt_ref, -) as w: - pass +) ``` ## YAML diff --git a/docs/hera_getting_started.md b/docs/hera_getting_started.md new file mode 100644 index 000000000..a79a96385 --- /dev/null +++ b/docs/hera_getting_started.md @@ -0,0 +1,28 @@ +# Hera + +## Quick Start + +```python +from hera.workflows import DAG, Container, Parameter, Workflow + +with Workflow( + generate_name="dag-diamond-", + entrypoint="diamond", +) as w: + echo = Container( + name="echo", + image="alpine:3.7", + command=["echo", "{{inputs.parameters.message}}"], + inputs=[Parameter(name="message")], + ) + with DAG(name="diamond"): + A = echo(name="A", arguments={"message": "A"}) + B = echo(name="B", arguments={"message": "B"}) + C = echo(name="C", arguments={"message": "C"}) + D = echo(name="D", arguments={"message": "D"}) + A >> [B, C] >> D + +w.create() +``` + +## Walk Through \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 15151f9d4..9cfe25e5f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,11 +6,14 @@ Welcome to Hera's documentation! ================================ +Hera is a Python framework for constructing and submitting Argo Workflows. The main goal of Hera is to make the Argo ecosystem accessible by simplifying workflow construction and submission. Hera presents an intuitive Python interface to the underlying API of Argo, with custom classes making use of context managers and callables, empowering users to focus on their own executable payloads rather than workflow setup. Hera allows power users of Argo to use Python if preferred, by ensuring feature parity with Argo and a fallback option through an OpenAPI generated Python module found at `hera.workflows.models`. + +See the Examples for usage and comparison to YAML. + .. toctree:: - :maxdepth: 2 - :caption: Hera + :caption: Getting Started - src + hera_getting_started .. toctree:: :caption: Expr Transpilation @@ -19,18 +22,30 @@ Welcome to Hera's documentation! .. toctree:: :glob: - :maxdepth: 2 + :maxdepth: 1 :caption: Hera Workflow Examples examples/workflows/* .. toctree:: :glob: - :maxdepth: 2 + :maxdepth: 1 :caption: Hera - Argo Workflow Examples Replication examples/workflows/upstream/* +.. toctree:: + :maxdepth: 3 + :caption: Hera Workflows API Reference + + hera.workflows + +.. toctree:: + :maxdepth: 3 + :caption: Hera Events API Reference + + hera.events + Indices and tables ================== diff --git a/docs/modules.rst b/docs/modules.rst deleted file mode 100644 index e9ff8ac1a..000000000 --- a/docs/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -src -=== - -.. toctree:: - :maxdepth: 4 - - src diff --git a/docs/requirements.txt b/docs/requirements.txt index e681d1451..038131d60 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ sphinx sphinx-book-theme +sphinx-autoapi myst-parser Jinja2<3.1 # -e . # hera itself diff --git a/docs/src.rst b/docs/src.rst deleted file mode 100644 index 8299fd2b5..000000000 --- a/docs/src.rst +++ /dev/null @@ -1,301 +0,0 @@ -hera -============ - -Submodules ----------- - -hera.affinity module --------------------- - -.. automodule:: hera.affinity - :members: - :undoc-members: - :show-inheritance: - -hera.archive module -------------------- - -.. automodule:: hera.archive - :members: - :undoc-members: - :show-inheritance: - -hera.artifact module --------------------- - -.. automodule:: hera.artifact - :members: - :undoc-members: - :show-inheritance: - -hera.backoff module -------------------- - -.. automodule:: hera.backoff - :members: - :undoc-members: - :show-inheritance: - -hera.client module ------------------- - -.. automodule:: hera.client - :members: - :undoc-members: - :show-inheritance: - -hera.config module ------------------- - -.. automodule:: hera.config - :members: - :undoc-members: - :show-inheritance: - -hera.cron\_workflow module --------------------------- - -.. automodule:: hera.cron_workflow - :members: - :undoc-members: - :show-inheritance: - -hera.dag module ---------------- - -.. automodule:: hera.dag - :members: - :undoc-members: - :show-inheritance: - -hera.env module ---------------- - -.. automodule:: hera.env - :members: - :undoc-members: - :show-inheritance: - -hera.env\_from module ---------------------- - -.. automodule:: hera.env_from - :members: - :undoc-members: - :show-inheritance: - -hera.host\_alias module ------------------------ - -.. automodule:: hera.host_alias - :members: - :undoc-members: - :show-inheritance: - -hera.host\_config module ------------------------- - -.. automodule:: hera.host_config - :members: - :undoc-members: - :show-inheritance: - -hera.image module ------------------ - -.. automodule:: hera.image - :members: - :undoc-members: - :show-inheritance: - -hera.io module --------------- - -.. automodule:: hera.io - :members: - :undoc-members: - :show-inheritance: - -hera.memoize module -------------------- - -.. automodule:: hera.memoize - :members: - :undoc-members: - :show-inheritance: - -hera.metric module ------------------- - -.. automodule:: hera.metric - :members: - :undoc-members: - :show-inheritance: - -hera.operator module --------------------- - -.. automodule:: hera.operator - :members: - :undoc-members: - :show-inheritance: - -hera.parameter module ---------------------- - -.. automodule:: hera.parameter - :members: - :undoc-members: - :show-inheritance: - -hera.resource\_template module ------------------------------- - -.. automodule:: hera.resource_template - :members: - :undoc-members: - :show-inheritance: - -hera.resources module ---------------------- - -.. automodule:: hera.resources - :members: - :undoc-members: - :show-inheritance: - -hera.retry\_policy module -------------------------- - -.. automodule:: hera.retry_policy - :members: - :undoc-members: - :show-inheritance: - -hera.retry\_strategy module ---------------------------- - -.. automodule:: hera.retry_strategy - :members: - :undoc-members: - :show-inheritance: - -hera.security\_context module ------------------------------ - -.. automodule:: hera.security_context - :members: - :undoc-members: - :show-inheritance: - -hera.sequence module --------------------- - -.. automodule:: hera.sequence - :members: - :undoc-members: - :show-inheritance: - -hera.task module ----------------- - -.. automodule:: hera.task - :members: - :undoc-members: - :show-inheritance: - -hera.template\_ref module -------------------------- - -.. automodule:: hera.template_ref - :members: - :undoc-members: - :show-inheritance: - -hera.toleration module ----------------------- - -.. automodule:: hera.toleration - :members: - :undoc-members: - :show-inheritance: - -hera.ttl\_strategy module -------------------------- - -.. automodule:: hera.ttl_strategy - :members: - :undoc-members: - :show-inheritance: - -hera.validators module ----------------------- - -.. automodule:: hera.validators - :members: - :undoc-members: - :show-inheritance: - -hera.value\_from module ------------------------ - -.. automodule:: hera.value_from - :members: - :undoc-members: - :show-inheritance: - -hera.volume\_claim\_gc module ------------------------------ - -.. automodule:: hera.volume_claim_gc - :members: - :undoc-members: - :show-inheritance: - -hera.volumes module -------------------- - -.. automodule:: hera.volumes - :members: - :undoc-members: - :show-inheritance: - -hera.workflow module --------------------- - -.. automodule:: hera.workflow - :members: - :undoc-members: - :show-inheritance: - -hera.workflow\_service module ------------------------------ - -.. automodule:: hera.workflow_service - :members: - :undoc-members: - :show-inheritance: - -hera.workflow\_status module ----------------------------- - -.. automodule:: hera.workflow_status - :members: - :undoc-members: - :show-inheritance: - -hera.workflow\_template module ------------------------------- - -.. automodule:: hera.workflow_template - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: hera - :members: - :undoc-members: - :show-inheritance: diff --git a/examples/workflows/upstream/cluster-workflow-template--workflow-template-ref.upstream.yaml b/examples/workflows/upstream/cluster-workflow-template--workflow-template-ref.upstream.yaml new file mode 100644 index 000000000..ab54992ad --- /dev/null +++ b/examples/workflows/upstream/cluster-workflow-template--workflow-template-ref.upstream.yaml @@ -0,0 +1,9 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: cluster-workflow-template-hello-world- +spec: + workflowTemplateRef: + name: cluster-workflow-template-submittable + clusterScope: true + diff --git a/examples/workflows/upstream/cluster-workflow-template--workflow-template-ref.yaml b/examples/workflows/upstream/cluster-workflow-template--workflow-template-ref.yaml new file mode 100644 index 000000000..4b6cd7e50 --- /dev/null +++ b/examples/workflows/upstream/cluster-workflow-template--workflow-template-ref.yaml @@ -0,0 +1,8 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: cluster-workflow-template-hello-world- +spec: + workflowTemplateRef: + clusterScope: true + name: cluster-workflow-template-submittable diff --git a/examples/workflows/upstream/cluster_workflow_template__workflow_template_ref.py b/examples/workflows/upstream/cluster_workflow_template__workflow_template_ref.py new file mode 100644 index 000000000..c9deb3b28 --- /dev/null +++ b/examples/workflows/upstream/cluster_workflow_template__workflow_template_ref.py @@ -0,0 +1,9 @@ +from hera.workflows import Workflow +from hera.workflows.models import WorkflowTemplateRef + +wt_ref = WorkflowTemplateRef(name="cluster-workflow-template-submittable", cluster_scope=True) + +w = Workflow( + generate_name="cluster-workflow-template-hello-world-", + workflow_template_ref=wt_ref, +) diff --git a/examples/workflows/upstream/workflow_template__workflow_template_ref.py b/examples/workflows/upstream/workflow_template__workflow_template_ref.py index a9fd57a4b..63ab32952 100644 --- a/examples/workflows/upstream/workflow_template__workflow_template_ref.py +++ b/examples/workflows/upstream/workflow_template__workflow_template_ref.py @@ -3,8 +3,7 @@ wt_ref = WorkflowTemplateRef(name="workflow-template-submittable") -with Workflow( +w = Workflow( generate_name="workflow-template-hello-world-", workflow_template_ref=wt_ref, -) as w: - pass +) diff --git a/scripts/init_files.py b/scripts/init_files.py index 591bcd449..52ba4d23a 100644 --- a/scripts/init_files.py +++ b/scripts/init_files.py @@ -10,14 +10,16 @@ if name != "models" and not name.startswith("_") ] -header = """ +header = ''' +"""Hera classes +""" # [DO NOT EDIT MANUALLY] # Auto-generated by Hera via `make init-files` # In order to add objects to the hera.workflows namespace # add them to the __all__ list in the relevant module. # Hera submodules should not use `from hera.workflows import X` # themselves, as it introduces a circular dependency. -""" +''' hera_workflows_init = Path(hera_workflows.__file__) diff --git a/scripts/models.py b/scripts/models.py index 3b6f8b332..fafb2fe70 100644 --- a/scripts/models.py +++ b/scripts/models.py @@ -97,10 +97,11 @@ def write_imports(imports: list, models_type: str, openapi_spec_url: str) -> Non path = Path(__name__).parent.parent / "src" / "hera" / models_type / "models" / "__init__.py" with open(str(path), "w+") as f: f.write( - f'"""[DO NOT EDIT MANUALLY]\n\n' - f"Auto-generated by Hera via `make {models_type}-models`\n" - f"OpenAPI spec URL: {openapi_spec_url}\n" + f'"""Auto-generated model classes\n' f'"""\n\n' + f"# [DO NOT EDIT MANUALLY]\n\n" + f"# Auto-generated by Hera via `make {models_type}-models`\n" + f"# OpenAPI spec URL: {openapi_spec_url}\n\n" ) for imp in imports: f.write(f"{imp}\n") diff --git a/src/hera/__init__.py b/src/hera/__init__.py index 0a03d2d56..850e1b2f0 100644 --- a/src/hera/__init__.py +++ b/src/hera/__init__.py @@ -1,3 +1,11 @@ +"""Hera is a Python framework for constructing and submitting Argo Workflows. +The main goal of Hera is to make the Argo ecosystem accessible by simplifying +workflow construction and submission. Hera presents an intuitive Python interface +to the underlying API of Argo, with custom classes making use of context managers +and callables, empowering users to focus on their own executable payloads rather +than workflow setup. +""" + from hera._version import version __version__ = version diff --git a/src/hera/events/models/__init__.py b/src/hera/events/models/__init__.py index 0e0866051..cfb50f684 100644 --- a/src/hera/events/models/__init__.py +++ b/src/hera/events/models/__init__.py @@ -1,9 +1,11 @@ -"""[DO NOT EDIT MANUALLY] - -Auto-generated by Hera via `make events-models` -OpenAPI spec URL: https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json +"""Auto-generated model classes """ +# [DO NOT EDIT MANUALLY] + +# Auto-generated by Hera via `make events-models` +# OpenAPI spec URL: https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json + from hera.events.models.eventsource import ( CreateEventSourceRequest, EventSourceDeletedResponse, diff --git a/src/hera/workflows/__init__.py b/src/hera/workflows/__init__.py index bbf570607..f6f578972 100644 --- a/src/hera/workflows/__init__.py +++ b/src/hera/workflows/__init__.py @@ -1,3 +1,5 @@ +"""Hera classes +""" # [DO NOT EDIT MANUALLY] # Auto-generated by Hera via `make init-files` # In order to add objects to the hera.workflows namespace diff --git a/src/hera/workflows/_mixins.py b/src/hera/workflows/_mixins.py index fd9f61963..200c9e4dc 100644 --- a/src/hera/workflows/_mixins.py +++ b/src/hera/workflows/_mixins.py @@ -493,7 +493,13 @@ def _build_inputs(self) -> Optional[ModelInputs]: return inputs -class TemplateInvocatorMixin(BaseMixin): +class TemplateInvocatorSubNodeMixin(BaseMixin): + """Used for classes that form sub nodes of Template Invocators - "Steps" and "DAG". + + See https://argoproj.github.io/argo-workflows/workflow-concepts/#template-invocators for + more on template invocators + """ + name: str continue_on: Optional[ContinueOn] = None hooks: Optional[Dict[str, LifecycleHook]] = None diff --git a/src/hera/workflows/artifact.py b/src/hera/workflows/artifact.py index 65cda2d96..3c441ac53 100644 --- a/src/hera/workflows/artifact.py +++ b/src/hera/workflows/artifact.py @@ -1,3 +1,9 @@ +"""The artifact module provides the base Artifact class, along with the various +types of artifacts as subclasses. + +See https://argoproj.github.io/argo-workflows/walk-through/artifacts/ +for a tutorial on Artifacts. +""" from typing import List, Optional, Union, cast from hera.shared._base_model import BaseModel diff --git a/src/hera/workflows/container.py b/src/hera/workflows/container.py index b83c6304e..e69b67c1d 100644 --- a/src/hera/workflows/container.py +++ b/src/hera/workflows/container.py @@ -1,3 +1,8 @@ +"""The container module provides the Container class. + +See https://argoproj.github.io/argo-workflows/workflow-concepts/#container +for more on containers. +""" from __future__ import annotations from typing import List, Optional @@ -26,6 +31,8 @@ class Container( VolumeMountMixin, CallableTemplateMixin, ): + """The Container template type defines a container to run on Argo.""" + args: Optional[List[str]] = None command: Optional[List[str]] = None lifecycle: Optional[Lifecycle] = None diff --git a/src/hera/workflows/cron_workflow.py b/src/hera/workflows/cron_workflow.py index 448bf348e..a82e4022b 100644 --- a/src/hera/workflows/cron_workflow.py +++ b/src/hera/workflows/cron_workflow.py @@ -1,6 +1,10 @@ +"""The cron_workflow module provides the CronWorkflow class + +See https://argoproj.github.io/argo-workflows/cron-workflows +for more on CronWorkflows. +""" from __future__ import annotations -from types import ModuleType from typing import Optional from hera.workflows.models import ( @@ -14,16 +18,15 @@ from hera.workflows.protocol import TWorkflow from hera.workflows.workflow import Workflow -_yaml: Optional[ModuleType] = None -try: - import yaml - _yaml = yaml -except ImportError: - _yaml = None +class CronWorkflow(Workflow): + """CronWorkflow allows a user to run a Workflow on a recurring basis. + NB: Hera's CronWorkflow is a subclass of Workflow which means certain fields are renamed + for compatibility, see `cron_suspend` and `cron_status` which are different from the Argo + spec. See https://argoproj.github.io/argo-workflows/fields/#cronworkflow + """ -class CronWorkflow(Workflow): concurrency_policy: Optional[str] = None failed_jobs_history_limit: Optional[int] = None schedule: str @@ -34,6 +37,7 @@ class CronWorkflow(Workflow): cron_status: Optional[CronWorkflowStatus] = None def build(self) -> TWorkflow: + """Builds the CronWorkflow and its components into an Argo schema CronWorkflow object.""" self = self._dispatch_hooks() return _ModelCronWorkflow( @@ -72,6 +76,7 @@ def build(self) -> TWorkflow: ) def create(self) -> TWorkflow: + """Creates the CronWorkflow on the Argo cluster.""" assert self.workflows_service, "workflow service not initialized" assert self.namespace, "workflow namespace not defined" return self.workflows_service.create_cron_workflow( @@ -79,6 +84,7 @@ def create(self) -> TWorkflow: ) def lint(self) -> TWorkflow: + """Lints the CronWorkflow using the Argo cluster.""" assert self.workflows_service, "workflow service not initialized" assert self.namespace, "workflow namespace not defined" return self.workflows_service.lint_cron_workflow( diff --git a/src/hera/workflows/dag.py b/src/hera/workflows/dag.py index fad4faec8..d729c46d3 100644 --- a/src/hera/workflows/dag.py +++ b/src/hera/workflows/dag.py @@ -1,3 +1,8 @@ +"""The dag module provides the DAG class. + +See https://argoproj.github.io/argo-workflows/walk-through/dag/ +for more on DAGs (Directed Acyclic Graphs). +""" from __future__ import annotations from typing import Any, List, Optional, Union @@ -13,6 +18,12 @@ class DAG(IOMixin, TemplateMixin, ContextMixin): + """A DAG template invocator is used to define Task dependencies as an acyclic graph. + + DAG implements the contextmanager interface so allows usage of `with`, under which any + `hera.workflows.task.Task` objects instantiated will be added to the DAG's list of Tasks. + """ + fail_fast: Optional[bool] = None target: Optional[str] = None tasks: List[Union[Task, DAGTask]] = [] diff --git a/src/hera/workflows/models/__init__.py b/src/hera/workflows/models/__init__.py index 126989104..0cf845122 100644 --- a/src/hera/workflows/models/__init__.py +++ b/src/hera/workflows/models/__init__.py @@ -1,9 +1,11 @@ -"""[DO NOT EDIT MANUALLY] - -Auto-generated by Hera via `make workflows-models` -OpenAPI spec URL: https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json +"""Auto-generated model classes """ +# [DO NOT EDIT MANUALLY] + +# Auto-generated by Hera via `make workflows-models` +# OpenAPI spec URL: https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json + from hera.workflows.models.google.protobuf import Any from hera.workflows.models.grpc.gateway.runtime import Error, StreamError from hera.workflows.models.io.argoproj.workflow.v1alpha1 import ( diff --git a/src/hera/workflows/parameter.py b/src/hera/workflows/parameter.py index c7484a671..df8ebd135 100644 --- a/src/hera/workflows/parameter.py +++ b/src/hera/workflows/parameter.py @@ -1,4 +1,8 @@ -"""Holds input model specifications""" +"""The parameter module provides the Parameter class. + +See https://argoproj.github.io/argo-workflows/walk-through/parameters/ +for a tutorial on Parameters. +""" from typing import Any, Optional from pydantic import root_validator @@ -8,6 +12,12 @@ class Parameter(_ModelParameter): + """A `Parameter` is used to pass values in and out of templates. + + They are to declare input and output parameters in the case of templates, and are used + for Steps and Tasks to assign values. + """ + value: Optional[Any] @root_validator(pre=True) diff --git a/src/hera/workflows/script.py b/src/hera/workflows/script.py index 283a71724..602b13c42 100644 --- a/src/hera/workflows/script.py +++ b/src/hera/workflows/script.py @@ -1,3 +1,8 @@ +"""The script module provides the Script class. + +See https://argoproj.github.io/argo-workflows/workflow-concepts/#script +for more on scripts. +""" import copy import inspect import textwrap @@ -30,6 +35,10 @@ class Script( ResourceMixin, VolumeMountMixin, ): + """A Script acts as a wrapper around a container. In Hera this defaults to a "python:3.7" image + specified by global_config.image, which runs a python source specified by `Script.source`. + """ + container_name: Optional[str] = None args: Optional[List[str]] = None command: Optional[List[str]] = global_config.script_command diff --git a/src/hera/workflows/steps.py b/src/hera/workflows/steps.py index b742211bb..4ae39e11e 100644 --- a/src/hera/workflows/steps.py +++ b/src/hera/workflows/steps.py @@ -1,3 +1,8 @@ +"""The steps module provides the Steps, Step and Parallel classes. + +See https://argoproj.github.io/argo-workflows/walk-through/steps/ +for more on Steps. +""" from typing import Any, List, Optional, Union from hera.workflows._mixins import ( @@ -7,7 +12,7 @@ ItemMixin, ParameterMixin, SubNodeMixin, - TemplateInvocatorMixin, + TemplateInvocatorSubNodeMixin, TemplateMixin, ) from hera.workflows.exceptions import InvalidType @@ -19,12 +24,16 @@ class Step( - TemplateInvocatorMixin, + TemplateInvocatorSubNodeMixin, ArgumentsMixin, SubNodeMixin, ParameterMixin, ItemMixin, ): + """Step is used to run a given template. Must be instantiated under a Steps or Parallel context, + or outside of a Workflow. + """ + @property def id(self) -> str: return f"{{{{steps.{self.name}.id}}}}" @@ -91,6 +100,12 @@ class Parallel( ContextMixin, SubNodeMixin, ): + """Parallel is used to add a list of steps which will run in parallel. + + Parallel implements the contextmanager interface so allows usage of `with`, under which any + `hera.workflows.steps.Step` objects instantiated will be added to Parallel's list of sub_steps. + """ + sub_steps: List[Union[Step, _ModelWorkflowStep]] = [] def _add_sub(self, node: Any): @@ -115,6 +130,17 @@ class Steps( IOMixin, TemplateMixin, ): + """A Steps template invocator is used to define a sequence of steps which can run + sequentially or in parallel. + + Steps implements the contextmanager interface so allows usage of `with`, under which any + `hera.workflows.steps.Step` objects instantiated will be added to the Steps' list of sub_steps. + + * Step and Parallel objects initialised within a Steps context will be added to the list of sub_steps + in the order they are initialised. + * All Step objects initialised within a Parallel context will run in parallel. + """ + sub_steps: List[ Union[ Step, @@ -154,6 +180,7 @@ def _add_sub(self, node: Any): self.sub_steps.append(node) def parallel(self) -> Parallel: + """Returns a Parallel object which can be used in a sub-context manager.""" return Parallel() def _build_template(self) -> _ModelTemplate: diff --git a/src/hera/workflows/suspend.py b/src/hera/workflows/suspend.py index c6b74a34b..477d265cd 100644 --- a/src/hera/workflows/suspend.py +++ b/src/hera/workflows/suspend.py @@ -1,3 +1,13 @@ +"""The suspend module provides the Suspend class. The Suspend template in Hera +provides a convenience wrapper around the "Intermediate Parameters" Argo feature. + +See https://argoproj.github.io/argo-workflows/walk-through/suspending/ +for more on suspending. + +See https://argoproj.github.io/argo-workflows/intermediate-inputs/ +for more on intermediate parameters. + +""" from typing import List, Optional, Union from hera.workflows._mixins import TemplateMixin @@ -11,6 +21,13 @@ class Suspend(TemplateMixin): + """The Suspend template allows the user to pause a workflow for a specified length of time + given by `duration` or indefinitely (i.e. until manually resumed). The Suspend template also + allows you to specify `intermediate_parameters` which will replicate the given parameters to + the "inputs" and "outputs" of the template, resulting in a Suspend template that pauses and + waits for values from the user for the given list of parameters. + """ + duration: Optional[Union[int, str]] = None intermediate_parameters: List[Parameter] = [] diff --git a/src/hera/workflows/task.py b/src/hera/workflows/task.py index d3bc6721e..670a23878 100644 --- a/src/hera/workflows/task.py +++ b/src/hera/workflows/task.py @@ -1,3 +1,8 @@ +"""The task module provides the Task and TaskResult classes. + +See https://argoproj.github.io/argo-workflows/walk-through/dag/ +for more on using Tasks within a DAG. +""" from __future__ import annotations from enum import Enum @@ -8,7 +13,7 @@ ItemMixin, ParameterMixin, SubNodeMixin, - TemplateInvocatorMixin, + TemplateInvocatorSubNodeMixin, TemplateMixin, ) from hera.workflows.models import ( @@ -21,6 +26,10 @@ class TaskResult(Enum): + """The enumeration of Task Results specified at + https://argoproj.github.io/argo-workflows/enhanced-depends-logic/#depends + """ + failed = "Failed" succeeded = "Succeeded" errored = "Errored" @@ -32,12 +41,67 @@ class TaskResult(Enum): class Task( - TemplateInvocatorMixin, + TemplateInvocatorSubNodeMixin, ArgumentsMixin, SubNodeMixin, ParameterMixin, ItemMixin, ): + r"""Task is used to run a given template within a DAG. Must be instantiated under a DAG context. + + ## Dependencies + Any Tasks without a dependency defined will start immediately. + + Dependencies between Tasks can be described using the convenience syntax `>>`, for example: + A = Task(...) + B = Task(...) + A >> B + describes the relationships: + * "A has no dependencies (so starts immediately) + * "B depends on A". + As a diagram: + A + | + B + + `A >> B` is equivalent to `A.next(B)`. + + ## Lists of Tasks + A list of Tasks used with the rshift syntax describes an "AND" dependency between the single Task on the left of + `>>` and the list Tasks to the right of `>>` (or vice versa). A list of Tasks on both sides of `>>` is not supported. + For example: + A = Task(...) + B = Task(...) + C = Task(...) + D = Task(...) + A >> [B, C] >> D + describes the relationships: + * "A has no dependencies + * "B AND C depend on A" + * "D depends on B AND C" + As a diagram: + A + / \ + B C + \ / + D + + Dependencies can be described over multiple statements: + A = Task(...) + B = Task(...) + C = Task(...) + D = Task(...) + A >> [C, D] + B >> [C, D] + describes the relationships: + * "A and B have no dependencies + * "C depends on A AND B" + * "D depends on A AND B" + As a diagram: + A B + | X | + C D + """ dependencies: Optional[List[str]] = None depends: Optional[str] = None @@ -82,6 +146,7 @@ def result(self) -> str: return f"{{{{tasks.{self.name}.outputs.result}}}}" def next(self, other: Task, operator: Operator = Operator.and_, on: Optional[TaskResult] = None) -> Task: + """Set self as a dependency of `other`.""" assert issubclass(other.__class__, Task) condition = f".{on}" if on else "" @@ -97,12 +162,14 @@ def next(self, other: Task, operator: Operator = Operator.and_, on: Optional[Tas return other def __rrshift__(self, other: List[Task]) -> Task: + """Set `other` as a dependency self.""" assert isinstance(other, list), f"Unknown type {type(other)} specified using reverse right bitshift operator" for o in other: o.next(self) return self def __rshift__(self, other: Union["Task", List["Task"]]) -> Union[Task, List[Task]]: + """Set self as a dependency of `other` which can be a single Task or list of Tasks.""" if isinstance(other, Task): return self.next(other) elif isinstance(other, list): diff --git a/src/hera/workflows/workflow.py b/src/hera/workflows/workflow.py index d6fcfa96d..b4d7a3438 100644 --- a/src/hera/workflows/workflow.py +++ b/src/hera/workflows/workflow.py @@ -1,3 +1,8 @@ +"""The workflow module provides the Workflow class + +See https://argoproj.github.io/argo-workflows/workflow-concepts/#the-workflow +for more on Workflows. +""" from __future__ import annotations from types import ModuleType @@ -60,6 +65,17 @@ class Workflow( ContextMixin, HookMixin, ): + """The base Workflow class for Hera. + + Workflow implements the contextmanager interface so allows usage of `with`, under which + any `hera.workflows.protocol.Templatable` object or `hera.workflows.models.Template` object + instantiated under the context will be added to the Workflow's list of templates. + + Workflows can be created directly on your Argo cluster via `create`. They can also be dumped + to yaml via `to_yaml` or built according to the Argo schema via `build` to get an OpenAPI model + object. + """ + # Workflow fields - https://argoproj.github.io/argo-workflows/fields/#workflow api_version: Optional[str] = global_config.api_version kind: Optional[str] = None @@ -142,6 +158,7 @@ def _set_kind(cls, v): return v def build(self) -> TWorkflow: + """Builds the Workflow and its components into an Argo schema Workflow object.""" self = self._dispatch_hooks() templates = [] @@ -254,24 +271,31 @@ def build(self) -> TWorkflow: ) def to_dict(self) -> Any: + """Builds the Workflow as an Argo schema Workflow object and returns it as a dictionary.""" return self.build().dict(exclude_none=True, by_alias=True) def to_yaml(self, *args, **kwargs) -> str: - if not yaml: + """Builds the Workflow as an Argo schema Workflow object and returns it as yaml string.""" + if not _yaml: raise ImportError("PyYAML is not installed") - return yaml.dump(self.to_dict(), *args, **kwargs) + return _yaml.dump(self.to_dict(), *args, **kwargs) def create(self) -> TWorkflow: + """Creates the Workflow on the Argo cluster.""" assert self.workflows_service, "workflow service not initialized" assert self.namespace, "workflow namespace not defined" return self.workflows_service.create_workflow(self.namespace, WorkflowCreateRequest(workflow=self.build())) def lint(self) -> TWorkflow: + """Lints the Workflow using the Argo cluster.""" assert self.workflows_service, "workflow service not initialized" assert self.namespace, "workflow namespace not defined" return self.workflows_service.lint_workflow(self.namespace, WorkflowLintRequest(workflow=self.build())) def _add_sub(self, node: Any): + """Adds any objects instantiated under the Workflow context manager that conform to the `Templatable` protocol + or are Argo schema Template objects to the Workflow's list of templates. + """ if not isinstance(node, (Templatable, *get_args(Template))): raise InvalidType() self.templates.append(node) diff --git a/src/hera/workflows/workflow_template.py b/src/hera/workflows/workflow_template.py index 1f05fab97..f5209fabc 100644 --- a/src/hera/workflows/workflow_template.py +++ b/src/hera/workflows/workflow_template.py @@ -1,3 +1,8 @@ +"""The workflow_template module provides the WorkflowTemplate class + +See https://argoproj.github.io/argo-workflows/workflow-templates/ +for more on WorkflowTemplates. +""" from pydantic import validator from typing_extensions import get_args @@ -15,6 +20,11 @@ class WorkflowTemplate(Workflow): + """WorkflowTemplates are definitions of Workflows that live in your cluster. This allows you + to create a library of frequently-used templates and reuse them by referencing them from your + Workflows. + """ + # WorkflowTemplate fields match Workflow exactly except for `status`, which WorkflowTemplate # does not have - https://argoproj.github.io/argo-workflows/fields/#workflowtemplate @validator("status", pre=True, always=True) @@ -23,6 +33,7 @@ def _set_status(cls, v): raise ValueError("status is not a valid field on a WorkflowTemplate") def create(self) -> TWorkflow: + """Creates the WorkflowTemplate on the Argo cluster.""" assert self.workflows_service, "workflow service not initialized" assert self.namespace, "workflow namespace not defined" return self.workflows_service.create_workflow_template( @@ -30,6 +41,7 @@ def create(self) -> TWorkflow: ) def lint(self) -> TWorkflow: + """Lints the WorkflowTemplate using the Argo cluster.""" assert self.workflows_service, "workflow service not initialized" assert self.namespace, "workflow namespace not defined" return self.workflows_service.lint_workflow_template( @@ -37,6 +49,7 @@ def lint(self) -> TWorkflow: ) def build(self) -> TWorkflow: + """Builds the WorkflowTemplate and its components into an Argo schema WorkflowTemplate object.""" self = self._dispatch_hooks() templates = []