diff --git a/docs/conf.py b/docs/conf.py index ebd887f4a5..9a8bbcaf0d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,6 +16,7 @@ from pathlib import Path import logging import sys +import inspect import sphinx.application import sphinx.errors @@ -58,8 +59,8 @@ "sphinx.ext.graphviz", "sphinx.ext.todo", "sphinx.ext.coverage", + "sphinx.ext.linkcode", "sphinx.ext.ifconfig", - "sphinx.ext.viewcode", "sphinx.ext.extlinks", "sphinx-prompt", "sphinx_copybutton", @@ -725,6 +726,67 @@ def filter(self, record: logging.LogRecord) -> bool: return True +def linkcode_resolve(domain, info): + """ + Determine the URL corresponding to Python object + """ + if domain != "py": + return None + + import flytekit + + modname = info["module"] + fullname = info["fullname"] + + submod = sys.modules.get(modname) + if submod is None: + return None + + obj = submod + for part in fullname.split("."): + try: + obj = getattr(obj, part) + except AttributeError: + return None + + if inspect.isfunction(obj): + obj = inspect.unwrap(obj) + try: + fn = inspect.getsourcefile(obj) + except TypeError: + fn = None + if not fn or fn.endswith("__init__.py"): + try: + fn = inspect.getsourcefile(sys.modules[obj.__module__]) + except (TypeError, AttributeError, KeyError): + fn = None + if not fn: + return None + + try: + source, lineno = inspect.getsourcelines(obj) + except (OSError, TypeError): + lineno = None + + linespec = f"#L{lineno:d}-L{lineno + len(source) - 1:d}" if lineno else "" + + startdir = Path(flytekit.__file__).parent.parent + try: + fn = os.path.relpath(fn, start=startdir).replace(os.path.sep, "/") + except ValueError: + return None + + if not fn.startswith("flytekit/"): + return None + + if flytekit.__version__ == "dev": + tag = "master" + else: + tag = f"v{flytekit.__version__}" + + return f"https://github.com/flyteorg/flytekit/blob/{tag}/{fn}{linespec}" + + def setup(app: sphinx.application.Sphinx) -> None: """Setup root logger for Sphinx""" logger = logging.getLogger("sphinx") diff --git a/docs/user_guide/concepts/main_concepts/executions.rst b/docs/user_guide/concepts/main_concepts/executions.rst index b6ee602520..dd95c25dee 100644 --- a/docs/user_guide/concepts/main_concepts/executions.rst +++ b/docs/user_guide/concepts/main_concepts/executions.rst @@ -6,7 +6,7 @@ Executions .. tags:: Basic, Glossary -**Executions** are instances of workflows, nodes or tasks created in the system as a result of a user-requested execution or a scheduled execution. +**Executions** are instances of workflows, nodes or tasks created in the system as a result of a user-requested execution or a scheduled execution. Execution IDs are unique within a given project domain, ensuring that no two executions within the same domain can have the same ID. Typical Flow Using Flytectl --------------------------- diff --git a/docs/user_guide/data_types_and_io/dataclass.md b/docs/user_guide/data_types_and_io/dataclass.md index 4ec45cc278..bc7ae9a26d 100644 --- a/docs/user_guide/data_types_and_io/dataclass.md +++ b/docs/user_guide/data_types_and_io/dataclass.md @@ -12,11 +12,7 @@ Flytekit uses the [Mashumaro library](https://github.com/Fatal1ty/mashumaro) to serialize and deserialize dataclasses. :::{important} -If you're using Flytekit version below v1.10, you'll need to decorate with `@dataclass_json` using -`from dataclasses_json import dataclass_json` instead of inheriting from Mashumaro's `DataClassJSONMixin`. - -If you're using Flytekit version >= v1.11.1, you don't need to decorate with `@dataclass_json` or -inherit from Mashumaro's `DataClassJSONMixin`. +If you're using Flytekit version below v1.11.1, you will need to add `from dataclasses_json import dataclass_json` to your imports and decorate your dataclass with `@dataclass_json`. ::: ```{note} @@ -25,15 +21,21 @@ To clone and run the example code on this page, see the [Flytesnacks repo][flyte To begin, import the necessary dependencies: -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py +:caption: data_types_and_io/dataclass.py +:lines: 1-9 +``` + +Build your custom image with ImageSpec: +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py :caption: data_types_and_io/dataclass.py -:lines: 1-10 +:lines: 16-19 ``` ## Python types We define a `dataclass` with `int`, `str` and `dict` as the data types. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py :caption: data_types_and_io/dataclass.py :pyobject: Datum ``` @@ -46,18 +48,18 @@ All variables in a data class should be **annotated with their type**. Failure t Once declared, a dataclass can be returned as an output or accepted as an input. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py :caption: data_types_and_io/dataclass.py -:lines: 28-43 +:lines: 32-47 ``` ## Flyte types We also define a data class that accepts {std:ref}`StructuredDataset `, {std:ref}`FlyteFile ` and {std:ref}`FlyteDirectory `. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py :caption: data_types_and_io/dataclass.py -:lines: 47-84 +:lines: 51-88 ``` A data class supports the usage of data associated with Python types, data classes, @@ -65,22 +67,22 @@ flyte file, flyte directory and structured dataset. We define a workflow that calls the tasks created above. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py :caption: data_types_and_io/dataclass.py :pyobject: dataclass_wf ``` You can run the workflow locally as follows: -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py :caption: data_types_and_io/dataclass.py -:lines: 97-98 +:lines: 101-102 ``` To trigger a task that accepts a dataclass as an input with `pyflyte run`, you can provide a JSON file as an input: ``` pyflyte run \ - https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/data_types_and_io/data_types_and_io/dataclass.py \ + https://raw.githubusercontent.com/flyteorg/flytesnacks/cfb5ea3b0d0502ef7df1f2e14f4a0d9b78250b6a/examples/data_types_and_io/data_types_and_io/dataclass.py \ add --x dataclass_input.json --y dataclass_input.json ``` diff --git a/flyteadmin/pkg/workflowengine/impl/prepare_execution.go b/flyteadmin/pkg/workflowengine/impl/prepare_execution.go index d8f544213e..169cb15616 100644 --- a/flyteadmin/pkg/workflowengine/impl/prepare_execution.go +++ b/flyteadmin/pkg/workflowengine/impl/prepare_execution.go @@ -129,6 +129,9 @@ func PrepareFlyteWorkflow(data interfaces.ExecutionData, flyteWorkflow *v1alpha1 acceptAtWrapper := v1.NewTime(data.ExecutionParameters.AcceptedAt) flyteWorkflow.AcceptedAt = &acceptAtWrapper + // Add finalizer + flyteWorkflow.Finalizers = append(flyteWorkflow.Finalizers, "flyte-finalizer") + // add permissions from auth and security context. Adding permissions from auth would be removed once all clients // have migrated over to security context addPermissions(data.ExecutionParameters.ExecutionConfig.SecurityContext, diff --git a/flyteadmin/pkg/workflowengine/impl/prepare_execution_test.go b/flyteadmin/pkg/workflowengine/impl/prepare_execution_test.go index be659208b2..58d5fa3934 100644 --- a/flyteadmin/pkg/workflowengine/impl/prepare_execution_test.go +++ b/flyteadmin/pkg/workflowengine/impl/prepare_execution_test.go @@ -254,4 +254,5 @@ func TestPrepareFlyteWorkflow(t *testing.T) { OutputLocationPrefix: "s3://bucket/key", }, }) + assert.Equal(t, flyteWorkflow.Finalizers, []string{"flyte-finalizer"}) }