From 7c4aec9a7b2e3c54130725e2e336c77078e891d8 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 23 Mar 2020 16:16:54 -0400 Subject: [PATCH] adjust process builtin id retrieval + tmp fix pywps replacing file/data value cause by auto-fetch (geopython/pywps#526, #91) --- weaver/processes/builtin/__init__.py | 13 ++++++++----- weaver/processes/wps_package.py | 11 +++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/weaver/processes/builtin/__init__.py b/weaver/processes/builtin/__init__.py index e6f471b87..f6f4cc9ec 100644 --- a/weaver/processes/builtin/__init__.py +++ b/weaver/processes/builtin/__init__.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from weaver.typedefs import AnyDatabaseContainer, CWL # noqa: F401 from cwltool.context import RuntimeContext # noqa: F401 - from typing import AnyStr, Dict, Type, Union # noqa: F401 + from typing import Any, AnyStr, Dict, Type, Union # noqa: F401 LOGGER = logging.getLogger(__name__) @@ -136,7 +136,10 @@ def register_builtin_processes(container): class BuiltinProcessJobBase(CommandLineJob): def __init__(self, builder, joborder, make_path_mapper, requirements, hints, name): - self.process = [h.get("process") for h in hints][0] + process_hints = [h for h in hints if "process" in h] + if not process_hints or len(process_hints) != 1: + raise PackageNotFound("Could not extract referenced process in job.") + self.process = process_hints[0]["process"] super(BuiltinProcessJobBase, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name) def _validate_process(self): @@ -149,11 +152,11 @@ def _validate_process(self): raise PackageExecutionError("Invalid package is not of type '{}'".format(PROCESS_BUILTIN)) # pylint: disable=W0221,arguments-differ # naming using python like arguments - def run(self, runtime_context): - # type: (RuntimeContext) -> None + def run(self, runtime_context, **kwargs): + # type: (RuntimeContext, Any) -> None try: self._validate_process() - super(BuiltinProcessJobBase, self).run(runtime_context) + super(BuiltinProcessJobBase, self).run(runtime_context, **kwargs) except Exception as err: LOGGER.warning(u"Failed to run process:\n%s", err, exc_info=runtime_context.debug) self.output_callback({}, "permanentFail") diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 9b9dbd5d3..1c4b1d472 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -647,12 +647,12 @@ def _cwl2wps_io(io_info, io_select): if "format" in io_info: io_formats = [io_info["format"]] if isinstance(io_info["format"], six.string_types) else io_info["format"] kw["supported_formats"] = [Format(clean_mime_type_format(str(fmt))) for fmt in io_formats] - kw["mode"] = MODE.SIMPLE + kw["mode"] = MODE.SIMPLE # only validate the extension (not file contents) else: # we need to minimally add 1 format, otherwise empty list is evaluated as None by pywps # when "supported_formats" is None, the process's json property raises because of it cannot iterate formats kw["supported_formats"] = [DefaultFormat] - kw["mode"] = MODE.NONE + kw["mode"] = MODE.NONE # don't validate anything as default is only raw text if is_output: if io_type == "Directory": kw["as_reference"] = True @@ -1821,6 +1821,13 @@ def _handler(self, request, response): cwl_inputs[input_id] = [self.make_location_input(data, input_type, input_def) for data, input_def in zip(input_data, input_occurs)] else: + # FIXME: we don't want auto fetch because we pass down value, but this is PyWPS bug + # (https://github.com/geopython/pywps/issues/526) + # (https://github.com/crim-ca/weaver/issues/91) + # href are sometime already handled (pulled and staged locally), use it directly + # validate using the internal '_file' instead of 'file' otherwise we trigger the fetch + if input_i.file and os.path.isfile(input_i._file): # noqa:W0212 + input_data = input_i.file cwl_inputs[input_id] = self.make_location_input(input_data, input_type, input_i) elif isinstance(input_i, (LiteralInput, BoundingBoxInput)): cwl_inputs[input_id] = input_data