From 0a1505cf7e4be602b1b805ae39cb4f57ca60b42b Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 11:44:48 -0400 Subject: [PATCH 01/10] Add test case --- cwltool/context.py | 3 +++ cwltool/workflow_job.py | 7 +++++- tests/test_context.py | 47 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/cwltool/context.py b/cwltool/context.py index bb281fd88..8b6eb76f2 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -29,6 +29,7 @@ from .process import Process from .secrets import SecretStore from .software_requirements import DependenciesConfiguration + from .workflow_job import WorkflowJobStep class ContextBase: @@ -191,6 +192,8 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None self.validate_only: bool = False self.validate_stdout: Optional["SupportsWrite[str]"] = None + self.workflow_job_step_name_callback: Optional[Callable[[WorkflowJobStep, CWLObjectType], str]] = None + super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index b552641e1..368d1cc02 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -60,7 +60,12 @@ def job( ) -> JobsGeneratorType: runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - runtimeContext.name = shortname(self.id) + + if runtimeContext.workflow_job_step_name_callback is not None: + vfinputs = {shortname(k): v for k, v in joborder.items()} + runtimeContext.name = runtimeContext.workflow_job_step_name_callback(self, vfinputs) + else: + runtimeContext.name = shortname(self.id) _logger.info("[%s] start", self.name) diff --git a/tests/test_context.py b/tests/test_context.py index acb7dfbca..601f5b770 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,5 +1,7 @@ import subprocess import sys +import logging +from io import StringIO from cwltool.context import RuntimeContext from cwltool.factory import Factory @@ -26,3 +28,48 @@ def test_replace_default_stdout_stderr() -> None: assert echo(inp="foo") == {"out": "foo\n"} sys.stdout = original_stdout sys.stderr = original_stderr + +def test_workflow_job_step_name_callback() -> None: + """Test ability to hook custom workflow step naming""" + + stream = StringIO() + streamhandler = logging.StreamHandler(stream) + _logger = logging.getLogger("cwltool") + _logger.addHandler(streamhandler) + + try: + runtime_context = RuntimeContext() + + def step_name_hook(step, job): + print(step, job) + return "%s on %s" % (step.name, job.get("revtool_input", job.get("sorted_input")).get("basename")) + + runtime_context.workflow_job_step_name_callback = step_name_hook + + factory = Factory(None, None, runtime_context) + revsort = factory.make(get_data("tests/wf/revsort.cwl")) + + result = revsort(workflow_input={"class": "File", "location": "whale.txt", "format": "https://www.iana.org/assignments/media-types/text/plain"}) + + del result["sorted_output"]["location"] + + assert result == { + 'sorted_output': { + 'basename': 'output.txt ', + 'checksum': 'sha1$b9214658cc453331b62c2282b772a5c063dbd284', + 'class': 'File', + 'http://commonwl.org/cwltool#generation': 0, + 'nameext': '.txt', + 'nameroot': 'output', + 'size': 1111, + }, + } + + print(stream.getvalue()) + + assert ( + "foostep" + in stream.getvalue() + ) + finally: + _logger.removeHandler(streamhandler) From 05067bb9ecdf6ce1479365c99338a515bd5637bf Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 13:26:13 -0400 Subject: [PATCH 02/10] Fix mypy and format --- cwltool/context.py | 4 +++- tests/test_context.py | 49 ++++++++++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/cwltool/context.py b/cwltool/context.py index 8b6eb76f2..83dd5a190 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -192,7 +192,9 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None self.validate_only: bool = False self.validate_stdout: Optional["SupportsWrite[str]"] = None - self.workflow_job_step_name_callback: Optional[Callable[[WorkflowJobStep, CWLObjectType], str]] = None + self.workflow_job_step_name_callback: Optional[ + Callable[[WorkflowJobStep, CWLObjectType], str] + ] = None super().__init__(kwargs) if self.tmp_outdir_prefix == "": diff --git a/tests/test_context.py b/tests/test_context.py index 601f5b770..148021e6b 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -2,9 +2,12 @@ import sys import logging from io import StringIO +from typing import MutableMapping, cast from cwltool.context import RuntimeContext from cwltool.factory import Factory +from cwltool.utils import CWLObjectType, CWLOutputType +from cwltool.workflow_job import WorkflowJobStep from .util import get_data @@ -29,6 +32,7 @@ def test_replace_default_stdout_stderr() -> None: sys.stdout = original_stdout sys.stderr = original_stderr + def test_workflow_job_step_name_callback() -> None: """Test ability to hook custom workflow step naming""" @@ -40,36 +44,47 @@ def test_workflow_job_step_name_callback() -> None: try: runtime_context = RuntimeContext() - def step_name_hook(step, job): - print(step, job) - return "%s on %s" % (step.name, job.get("revtool_input", job.get("sorted_input")).get("basename")) + def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: + j1 = cast(MutableMapping[str, CWLObjectType], job) + inp = cast(MutableMapping[str, str], j1.get("revtool_input", j1.get("sorted_input"))) + return "%s on %s" % ( + step.name, + inp.get("basename"), + ) runtime_context.workflow_job_step_name_callback = step_name_hook factory = Factory(None, None, runtime_context) revsort = factory.make(get_data("tests/wf/revsort.cwl")) - result = revsort(workflow_input={"class": "File", "location": "whale.txt", "format": "https://www.iana.org/assignments/media-types/text/plain"}) + result = revsort( + workflow_input={ + "class": "File", + "location": "whale.txt", + "format": "https://www.iana.org/assignments/media-types/text/plain", + } + ) + + result = cast(CWLObjectType, result) - del result["sorted_output"]["location"] + sorted_out = cast(MutableMapping[str, str], result["sorted_output"]) + loc = sorted_out["location"] assert result == { - 'sorted_output': { - 'basename': 'output.txt ', - 'checksum': 'sha1$b9214658cc453331b62c2282b772a5c063dbd284', - 'class': 'File', - 'http://commonwl.org/cwltool#generation': 0, - 'nameext': '.txt', - 'nameroot': 'output', - 'size': 1111, + "sorted_output": { + "basename": "output.txt ", + "checksum": "sha1$b9214658cc453331b62c2282b772a5c063dbd284", + "class": "File", + "http://commonwl.org/cwltool#generation": 0, + "nameext": ".txt", + "nameroot": "output", + "size": 1111, + "location": loc, }, } print(stream.getvalue()) - assert ( - "foostep" - in stream.getvalue() - ) + assert "foostep" in stream.getvalue() finally: _logger.removeHandler(streamhandler) From 39c6107b0be0362d88b74de962d29d0c0825771c Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 13:58:35 -0400 Subject: [PATCH 03/10] Fix test --- tests/test_context.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_context.py b/tests/test_context.py index 148021e6b..5c718eea0 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -60,7 +60,7 @@ def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: result = revsort( workflow_input={ "class": "File", - "location": "whale.txt", + "location": "tests/wf/whale.txt", "format": "https://www.iana.org/assignments/media-types/text/plain", } ) @@ -72,7 +72,7 @@ def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: assert result == { "sorted_output": { - "basename": "output.txt ", + "basename": "output.txt", "checksum": "sha1$b9214658cc453331b62c2282b772a5c063dbd284", "class": "File", "http://commonwl.org/cwltool#generation": 0, @@ -83,8 +83,6 @@ def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: }, } - print(stream.getvalue()) - - assert "foostep" in stream.getvalue() + assert "rev on whale.txt" in stream.getvalue() finally: _logger.removeHandler(streamhandler) From 0b9ff78572cfc1d05e6361a2cb71d0195f2aa500 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 15:12:15 -0400 Subject: [PATCH 04/10] Fix test to use get_data --- tests/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_context.py b/tests/test_context.py index 5c718eea0..5890423a4 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -60,7 +60,7 @@ def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: result = revsort( workflow_input={ "class": "File", - "location": "tests/wf/whale.txt", + "location": get_data("tests/wf/whale.txt"), "format": "https://www.iana.org/assignments/media-types/text/plain", } ) From 690b6b2f6c460d0a9918519656f50bc25cbc7f4a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 15:21:43 -0400 Subject: [PATCH 05/10] flake8 fix --- tests/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_context.py b/tests/test_context.py index 5890423a4..d325be28a 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -6,7 +6,7 @@ from cwltool.context import RuntimeContext from cwltool.factory import Factory -from cwltool.utils import CWLObjectType, CWLOutputType +from cwltool.utils import CWLObjectType from cwltool.workflow_job import WorkflowJobStep from .util import get_data From cd8258fe8d80bc72bd0c087eb17fe978db89287b Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 15:32:30 -0400 Subject: [PATCH 06/10] revsort needs docker, oops. --- tests/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_context.py b/tests/test_context.py index d325be28a..4111dbd8f 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -32,7 +32,7 @@ def test_replace_default_stdout_stderr() -> None: sys.stdout = original_stdout sys.stderr = original_stderr - +@needs_docker def test_workflow_job_step_name_callback() -> None: """Test ability to hook custom workflow step naming""" From fb108d405fbafc15d55325e930d4b9d2093c6372 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 15:45:35 -0400 Subject: [PATCH 07/10] import needs_docker --- tests/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_context.py b/tests/test_context.py index 4111dbd8f..44782247e 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -9,7 +9,7 @@ from cwltool.utils import CWLObjectType from cwltool.workflow_job import WorkflowJobStep -from .util import get_data +from .util import get_data, needs_docker def test_replace_default_stdout_stderr() -> None: From cd461fdecf5c3b7bf219c41a522d30c9b1861d2f Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 16:19:26 -0400 Subject: [PATCH 08/10] uri prefix location properly Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- tests/test_context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_context.py b/tests/test_context.py index 44782247e..bbd147800 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -8,6 +8,7 @@ from cwltool.factory import Factory from cwltool.utils import CWLObjectType from cwltool.workflow_job import WorkflowJobStep +from pathlib import Path from .util import get_data, needs_docker @@ -60,7 +61,7 @@ def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: result = revsort( workflow_input={ "class": "File", - "location": get_data("tests/wf/whale.txt"), + "location": Path(get_data("tests/wf/whale.txt")).as_uri(), "format": "https://www.iana.org/assignments/media-types/text/plain", } ) From 2e7b39dff06a3198b477abaed8c228d0f366e3ec Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 16:29:35 -0400 Subject: [PATCH 09/10] fix flake8 --- tests/test_context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_context.py b/tests/test_context.py index bbd147800..be1d89575 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -33,6 +33,7 @@ def test_replace_default_stdout_stderr() -> None: sys.stdout = original_stdout sys.stderr = original_stderr + @needs_docker def test_workflow_job_step_name_callback() -> None: """Test ability to hook custom workflow step naming""" From e3094a6c910dd22e465d5240d6b6cf87494c45e4 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 24 Mar 2025 18:20:09 -0400 Subject: [PATCH 10/10] exclude test_workflow_job_step_name_callback from build_test_container --- build-cwltool-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index 9f7163afb..3dfa7bb66 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -8,4 +8,4 @@ ${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ - "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" + "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req or test_workflow_job_step_name_callback)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool"