From ec60adf15855ee5d0266b45fd8a85697c8b09f79 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 25 May 2020 21:45:25 -0400 Subject: [PATCH] workdir config + enforce existing cwl-io-formats (relates to #50) + log pywps config loading --- CHANGES.rst | 16 +++ Makefile | 3 +- tests/functional/test_wps_package.py | 105 +++++++++++++++++ weaver/formats.py | 13 ++- weaver/processes/wps_package.py | 134 +++++++++++++--------- weaver/wps.py | 43 ++++--- weaver/wps_restapi/jobs/jobs.py | 2 +- weaver/wps_restapi/processes/processes.py | 7 +- weaver/wps_restapi/swagger_definitions.py | 1 + 9 files changed, 243 insertions(+), 81 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7ef35a26d..4be3bbc9c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,22 @@ Changes `Unreleased `_ (latest) ======================================================================== +Changes: +-------- + +- Add ``weaver.wps_workdir`` configuration setting to define the location where the underlying ``cwltool`` application + should be executed under. +- Use ``weaver.request_options`` for `WPS GetCapabilities` and `WPS Check Status` requests under the running job. +- Change default ``DOCKER_REPO`` value defined in ``Makefile`` to point to reference mentioned in ``README.md`` and + considered as official deployment location. + +Fixes: +------ + +- Set ``get_cwl_file_format`` default argument ``must_exist=True`` instead of ``False`` to retrieve original default + behaviour of the function. Since `CWL` usually doesn't need to add ``File.format`` field when no corresponding + reference actually exists, this default also makes more sense. + `1.8.1 `_ (2020-05-22) ======================================================================== diff --git a/Makefile b/Makefile index 0e8600392..914a526af 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..) APP_NAME := $(shell basename $(APP_ROOT)) APP_VERSION ?= 1.8.1 APP_INI ?= $(APP_ROOT)/config/$(APP_NAME).ini -DOCKER_REPO ?= docker-registry.crim.ca/ogc/weaver +DOCKER_REPO ?= pavics/weaver +#DOCKER_REPO ?= docker-registry.crim.ca/ogc/weaver # guess OS (Linux, Darwin,...) OS_NAME := $(shell uname -s 2>/dev/null || echo "unknown") diff --git a/tests/functional/test_wps_package.py b/tests/functional/test_wps_package.py index baa33e395..33152d7aa 100644 --- a/tests/functional/test_wps_package.py +++ b/tests/functional/test_wps_package.py @@ -248,6 +248,111 @@ def test_literal_io_from_package_and_offering(self): # assert pkg["outputs"][1]["label"] == "Additional detail only within WPS output", \ # "WPS I/O title should be converted to CWL label of corresponding I/O from additional details" + def test_complex_io_format_references(self): + """ + Test validates that known `WPS` I/O formats (i.e.: `MIME-type`) considered as valid, but not corresponding + to any *real* `IANA/EDAM` reference for `CWL` are preserved on the `WPS` side and dropped on `CWL` side to + avoid validation error. + + We also validate a `MIME-type` that should be found for both `CWL` and `WPS` formats to make sure that `CWL` + formats are only dropped when necessary. + """ + ns_json, type_json = get_cwl_file_format(CONTENT_TYPE_APP_JSON, must_exist=True) + assert "iana" in ns_json # just to make sure + ct_not_exists = "x-ogc-dods" # OpenDAP, still doesn't exist at moment of test creation + ns_not_exists, type_not_exists = get_cwl_file_format(ct_not_exists, must_exist=False) + assert "iana" in ns_not_exists + body = { + "processDescription": { + "process": { + "id": self._testMethodName, + "inputs": [ + { + "id": "wps_only_format_exists", + "formats": [ + { + "mimeType": CONTENT_TYPE_APP_JSON, + "default": True, + } + ] + }, + { + "id": "wps_only_format_not_exists", + "formats": [ + { + "mimeType": ct_not_exists, + "default": True, + } + ] + }, + { + "id": "wps_only_format_both", + "formats": [ + {"mimeType": CONTENT_TYPE_APP_JSON}, + {"mimeType": ct_not_exists, "default": True}, + ] + } + ], + # NOTE: + # Don't care about outputs here since we cannot have an array of formats + # as CWL output, so there isn't much to compare against from the WPS list. + }, + }, + "deploymentProfileName": "http://www.opengis.net/profiles/eoc/wpsApplication", + "executionUnit": [{"unit": { + "cwlVersion": "v1.0", + "class": "CommandLineTool", + "inputs": { + # minimal info only to match IDs, check that formats are added only when CWL can resolve references + # FIXME: no format is back-propagated from WPS format to CWL at the moment + # (https://github.com/crim-ca/weaver/issues/50) + "wps_only_format_exists": "File", + "wps_only_format_not_exists": "File", + "wps_only_format_both": "File", + "cwl_only_format_exists": {"type": "File", "format": type_json}, + # non-existing schema references should not be provided directly in CWL + # since these would enforce raising the validation error directly... + # "cwl_only_format_not_exists": {"type": "File", "format": ct_not_exists} + }, + "outputs": {"dont_care": "File"}, + "$namespaces": dict(list(ns_json.items())) + }}], + } + desc, pkg = self.deploy_process(body) + + assert desc["process"]["inputs"][0]["id"] == "wps_only_format_exists" + assert len(desc["process"]["inputs"][0]["formats"]) == 1 + assert desc["process"]["inputs"][0]["formats"][0]["mimeType"] == CONTENT_TYPE_APP_JSON + assert pkg["inputs"][0]["id"] == "wps_only_format_exists" + assert pkg["inputs"][0]["type"] == "File" + # FIXME: back-propagate WPS format to CWL without format specified + # (https://github.com/crim-ca/weaver/issues/50) + # assert pkg["inputs"][0]["format"] == type_json + + assert desc["process"]["inputs"][1]["id"] == "wps_only_format_not_exists" + assert len(desc["process"]["inputs"][1]["formats"]) == 1 + assert desc["process"]["inputs"][1]["formats"][0]["mimeType"] == ct_not_exists + assert pkg["inputs"][1]["id"] == "wps_only_format_not_exists" + assert pkg["inputs"][1]["type"] == "File" + assert "format" not in pkg["inputs"][1], "Non-existing CWL format reference should have been dropped." + + assert desc["process"]["inputs"][2]["id"] == "wps_only_format_both" + assert len(desc["process"]["inputs"][2]["formats"]) == 2 + assert desc["process"]["inputs"][2]["formats"][0]["mimeType"] == CONTENT_TYPE_APP_JSON + assert desc["process"]["inputs"][2]["formats"][1]["mimeType"] == ct_not_exists + assert pkg["inputs"][2]["id"] == "wps_only_format_both" + assert pkg["inputs"][2]["type"] == "File" + # FIXME: for now we don't even back-propagate, but if we did, must be none because one is unknown reference + # (https://github.com/crim-ca/weaver/issues/50) + assert "format" not in pkg["inputs"][2], "Any non-existing CWL format reference should drop all entries." + + assert desc["process"]["inputs"][3]["id"] == "cwl_only_format_exists" + assert len(desc["process"]["inputs"][3]["formats"]) == 1 + assert desc["process"]["inputs"][3]["formats"][0]["mimeType"] == CONTENT_TYPE_APP_JSON + assert pkg["inputs"][3]["id"] == "cwl_only_format_exists" + assert pkg["inputs"][3]["type"] == "File" + assert pkg["inputs"][3]["format"] == type_json + def test_complex_io_with_multiple_formats_and_defaults(self): """ Test validates that different format types are set on different input variations simultaneously: diff --git a/weaver/formats.py b/weaver/formats.py index efab43e86..3682405a6 100644 --- a/weaver/formats.py +++ b/weaver/formats.py @@ -85,13 +85,13 @@ def get_extension(mime_type): FORMAT_NAMESPACES = frozenset([IANA_NAMESPACE, EDAM_NAMESPACE]) -def get_cwl_file_format(mime_type, make_reference=False, must_exist=False): +def get_cwl_file_format(mime_type, make_reference=False, must_exist=True): # type: (AnyStr, bool, bool) -> Union[Tuple[Union[JSON, None], Union[AnyStr, None]], Union[AnyStr, None]] """ Obtains the corresponding `IANA`/`EDAM` ``format`` value to be applied under a `CWL` I/O ``File`` from the ``mime_type`` (`Content-Type` header) using the first matched one. - If ``make_reference=False``: + - If ``make_reference=False``: - If there is a match, returns ``tuple({}, )``: 1) corresponding namespace mapping to be applied under ``$namespaces`` in the `CWL`. 2) value of ``format`` adjusted according to the namespace to be applied to ``File`` in the `CWL`. @@ -99,14 +99,17 @@ def get_cwl_file_format(mime_type, make_reference=False, must_exist=False): returns a literal and non-existing definition as ``tuple({"iana": }, )`` - Otherwise, returns ``(None, None)`` - If ``make_reference=True``: + - If ``make_reference=True``: - If there is a match, returns the explicit format reference as ``/``. - If there is no match but ``must_exist=False``, returns the literal reference as ``/``. - Otherwise, returns a single ``None``. Note: - In situations where ``must_exist=False`` and the default non-existing namespace is returned, the `CWL` - behaviour is to evaluate corresponding ``format`` for literal matching strings. + In situations where ``must_exist=False`` is used and that the namespace and/or full format URL cannot be + resolved to an existing reference, `CWL` will raise a validation error as it cannot confirm the ``format``. + You must therefore make sure that the returned reference really exists when using ``must_exist=False`` before + providing it to the `CWL` I/O definition. This parameter should be used only for literal string comparison or + pre-processing steps to evaluate formats. """ def _make_if_ref(_map, _key, _fmt): return os.path.join(_map[_key], _fmt) if make_reference else (_map, "{}:{}".format(_key, _fmt)) diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index e87cd2e99..af0e56a10 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -1422,7 +1422,7 @@ def _get_cwl_fmt_details(wps_fmt): if not _wps_io_fmt: return None, None, None _cwl_io_ext = get_extension(_wps_io_fmt) - _cwl_io_ref, _cwl_io_fmt = get_cwl_file_format(_wps_io_fmt) + _cwl_io_ref, _cwl_io_fmt = get_cwl_file_format(_wps_io_fmt, must_exist=True) return _cwl_io_ref, _cwl_io_fmt, _cwl_io_ext wps_io_type = _get_field(wps_io, "type", search_variations=True) @@ -1459,6 +1459,11 @@ def _get_cwl_fmt_details(wps_fmt): break # don't use any format because we cannot enforce one cwl_io_fmt = [] for fmt_i in fmt: + # FIXME: (?) + # when multiple formats are specified, but at least one schema/namespace reference can't be found, + # should we drop all since that unknown format is still allowed but cannot be validated? + # avoid potential validation error if that format was the one provided during execute... + # (see: https://github.com/crim-ca/weaver/issues/50) cwl_io_ref_i, cwl_io_fmt_i, _ = _get_cwl_fmt_details(fmt_i) if cwl_io_ref_i and cwl_io_fmt_i: cwl_io_fmt.append(cwl_io_fmt_i) @@ -1900,42 +1905,6 @@ def map_step_progress(cls, step_index, steps_total): """ return map_progress(100 * step_index / steps_total, PACKAGE_PROGRESS_CWL_RUN, PACKAGE_PROGRESS_CWL_DONE) - @staticmethod - def make_location_input(input_type, input_definition): - # type: (AnyStr, ComplexInput) -> JSON - """Generates the JSON content required to specify a CWL File input definition from a location.""" - # We don't want auto fetch because we pass down value to CWL which will handle it accordingly - input_location = None - # cannot rely only on 'as_reference' as sometime it is not provided by the request although it's an href - if input_definition.as_reference: - input_location = input_definition.url - # FIXME: PyWPS bug - calling 'file' method fetches it, and it is always called during type validation - # (https://github.com/geopython/pywps/issues/526) - # (https://github.com/crim-ca/weaver/issues/91) - # since href is already handled (pulled and staged locally), use it directly to avoid double fetch with CWL - # validate using the internal '_file' instead of 'file' otherwise we trigger the fetch - # normally, file should be pulled an this check should fail - if input_definition._file and os.path.isfile(input_definition._file): # noqa: W0212 - input_location = input_definition._file # noqa: W0212 - # if source type is data, we actually need to call 'data' (without fetch of remote file, already fetched) - # value of 'file' in this case points to a local file path where the wanted link was dumped as raw data - if input_definition.source_type == SOURCE_TYPE.DATA: - input_location = input_definition.data - if not input_location: - url = getattr(input_definition, "url") - if isinstance(url, six.string_types) and any([url.startswith(p) for p in ["http", "file"]]): - input_location = url - else: - # last option, could not resolve 'lazily' so will fetch data if needed - input_location = input_definition.data - - location = {"location": input_location, "class": input_type} - if input_definition.data_format is not None and input_definition.data_format.mime_type: - fmt = get_cwl_file_format(input_definition.data_format.mime_type, make_reference=True) - if fmt is not None: - location["format"] = fmt - return location - def _handler(self, request, response): # type: (WPSRequest, ExecuteResponse) -> ExecuteResponse LOGGER.debug("HOME=%s, Current Dir=%s", os.environ.get("HOME"), os.path.abspath(os.curdir)) @@ -1952,8 +1921,8 @@ def _handler(self, request, response): self.update_status("Launching package...", PACKAGE_PROGRESS_LAUNCHING, STATUS_RUNNING) - settings = get_settings(app) - is_ems = get_weaver_configuration(settings) == WEAVER_CONFIGURATION_EMS + self.settings = get_settings(app) + is_ems = get_weaver_configuration(self.settings) == WEAVER_CONFIGURATION_EMS if is_ems: # EMS dispatch the execution to the ADES loading_context = LoadingContext() @@ -1962,9 +1931,14 @@ def _handler(self, request, response): # ADES execute the cwl locally loading_context = None - wps_out_dir_prefix = os.path.join(get_wps_output_dir(settings), "tmp") - runtime_args = {"no_read_only": True, "outdir": self.workdir, "tmp_outdir_prefix": wps_out_dir_prefix} - runtime_context = RuntimeContext(kwargs=runtime_args) + ##wps_out_dir_prefix = os.path.join(get_wps_output_dir(settings), "tmp") + wps_workdir = self.settings.get("weaver.wps_workdir", self.workdir) + runtime_context = RuntimeContext(kwargs={ + "no_read_only": True, + "outdir": wps_workdir, # if using any other than docker app + "docker_outdir": wps_workdir, # if using docker app + #"tmp_outdir_prefix": wps_out_dir_prefix, ## FIXME: old-location + }) try: package_inst, _, self.step_packages = _load_package_content(self.package, package_name=self.package_id, @@ -1996,7 +1970,7 @@ def _handler(self, request, response): request.inputs = opensearch.query_eo_images_from_wps_inputs(request.inputs, eoimage_data_sources, accept_mime_types, - settings=settings) + settings=self.settings) cwl_inputs = dict() for input_id in request.inputs: @@ -2046,15 +2020,7 @@ def _handler(self, request, response): # FIXME: this won't be necessary using async routine (https://github.com/crim-ca/weaver/issues/131) self.insert_package_log(result) try: - for output in request.outputs: - # TODO: adjust output for glob patterns (https://github.com/crim-ca/weaver/issues/24) - if isinstance(result[output], list) and not isinstance(self.response.outputs[output], list): - result[output] = result[output][0] # expect only one output - if "location" in result[output]: - self.response.outputs[output].as_reference = True - self.response.outputs[output].file = result[output]["location"].replace("file://", "") - else: - self.response.outputs[output].data = result[output] + self.make_location_outputs(result) self.update_status("Generate package outputs done.", PACKAGE_PROGRESS_PREP_OUT, STATUS_RUNNING) except Exception as exc: raise self.exception_message(PackageExecutionError, exc, "Failed to save package outputs.") @@ -2067,6 +2033,70 @@ def _handler(self, request, response): self.update_status("Package complete.", PACKAGE_PROGRESS_DONE, STATUS_SUCCEEDED) return self.response + @staticmethod + def make_location_input(input_type, input_definition): + # type: (AnyStr, ComplexInput) -> JSON + """Generates the JSON content required to specify a CWL File input definition from a location.""" + # We don't want auto fetch because we pass down value to CWL which will handle it accordingly + input_location = None + # cannot rely only on 'as_reference' as sometime it is not provided by the request although it's an href + if input_definition.as_reference: + input_location = input_definition.url + # FIXME: PyWPS bug - calling 'file' method fetches it, and it is always called during type validation + # (https://github.com/geopython/pywps/issues/526) + # (https://github.com/crim-ca/weaver/issues/91) + # since href is already handled (pulled and staged locally), use it directly to avoid double fetch with CWL + # validate using the internal '_file' instead of 'file' otherwise we trigger the fetch + # normally, file should be pulled an this check should fail + if input_definition._file and os.path.isfile(input_definition._file): # noqa: W0212 + input_location = input_definition._file # noqa: W0212 + # if source type is data, we actually need to call 'data' (without fetch of remote file, already fetched) + # value of 'file' in this case points to a local file path where the wanted link was dumped as raw data + if input_definition.source_type == SOURCE_TYPE.DATA: + input_location = input_definition.data + if not input_location: + url = getattr(input_definition, "url") + if isinstance(url, six.string_types) and any([url.startswith(p) for p in ["http", "file"]]): + input_location = url + else: + # last option, could not resolve 'lazily' so will fetch data if needed + input_location = input_definition.data + + location = {"location": input_location, "class": input_type} + if input_definition.data_format is not None and input_definition.data_format.mime_type: + fmt = get_cwl_file_format(input_definition.data_format.mime_type, make_reference=True) + if fmt is not None: + location["format"] = fmt + return location + + def make_location_outputs(self, cwl_result): + """ + Maps `CWL` result outputs to corresponding `WPS` outputs under required location. + """ + wps_out_dir = get_wps_output_dir(self.settings) + for output_id in self.request.outputs: + # TODO: adjust output for glob patterns (https://github.com/crim-ca/weaver/issues/24) + if isinstance(cwl_result[output_id], list) and not isinstance(self.response.outputs[output_id], list): + cwl_result[output_id] = cwl_result[output_id][0] # expect only one output + if "location" in cwl_result[output_id]: + result_loc = cwl_result[output_id]["location"].replace("file://", "") + result_wps = os.path.join(wps_out_dir, os.path.split(result_loc)[-1]) + if os.path.realpath(result_loc) != os.path.realpath(result_wps): + LOGGER.info("Moving [%s]: [%s] -> [%s]", output_id, result_loc, result_wps) + shutil.move(result_loc, result_wps) + self.response.outputs[output_id].as_reference = True + self.response.outputs[output_id].file = result_wps + LOGGER.info("Resolved WPS output [%s]: [%s]", output_id, result_wps) + else: + result_loc = cwl_result[output_id] + result_wps = os.path.join(wps_out_dir, os.path.split(result_loc)[-1]) + if os.path.realpath(result_loc) != os.path.realpath(result_wps): + LOGGER.info("Moving: [%s] -> [%s]", result_loc, result_wps) + shutil.move(result_loc, result_wps) + LOGGER.info("Moving [%s]: [%s] -> [%s]", output_id, result_loc, result_wps) + self.response.outputs[output_id].data = result_wps + LOGGER.info("Resolved WPS output [%s]: [%s]", output_id, result_wps) + def make_tool(self, toolpath_object, loading_context): # type: (ToolPathObjectType, LoadingContext) -> ProcessCWL from weaver.processes.wps_workflow import default_make_tool diff --git a/weaver/wps.py b/weaver/wps.py index e68ea4a09..1ce5f790b 100644 --- a/weaver/wps.py +++ b/weaver/wps.py @@ -101,26 +101,31 @@ def get_wps_output_url(container): def load_pywps_cfg(container, config=None): # type: (AnySettingsContainer, Optional[Union[AnyStr, Dict[AnyStr, AnyStr]]]) -> ConfigParser - """Loads and updates the PyWPS configuration using Weaver settings.""" + """ + Loads and updates the PyWPS configuration using Weaver settings. + """ settings = get_settings(container) - if not settings.get("weaver.wps_configured"): - # initial setup of PyWPS config - pywps_config.load_configuration([]) # load defaults - # must be set to INFO to disable sqlalchemy trace. - # see : https://github.com/geopython/pywps/blob/master/pywps/dblog.py#L169 - if logging.getLevelName(pywps_config.CONFIG.get("logging", "level")) <= logging.DEBUG: - pywps_config.CONFIG.set("logging", "level", "INFO") - # update metadata - for setting_name, setting_value in settings.items(): - if setting_name.startswith("weaver.wps_metadata"): - pywps_setting = setting_name.replace("weaver.wps_metadata_", "") - pywps_config.CONFIG.set("metadata:main", pywps_setting, setting_value) - # add weaver configuration keyword if not already provided - wps_keywords = pywps_config.CONFIG.get("metadata:main", "identification_keywords") - weaver_mode = get_weaver_configuration(settings) - if weaver_mode not in wps_keywords: - wps_keywords += ("," if wps_keywords else "") + weaver_mode - pywps_config.CONFIG.set("metadata:main", "identification_keywords", wps_keywords) + if settings.get("weaver.wps_configured"): + LOGGER.debug("Using preloaded internal Weaver WPS configuration.") + return pywps_config.CONFIG + + LOGGER.info("Initial load of internal Weaver WPS configuration.") + pywps_config.load_configuration([]) # load defaults + # must be set to INFO to disable sqlalchemy trace. + # see : https://github.com/geopython/pywps/blob/master/pywps/dblog.py#L169 + if logging.getLevelName(pywps_config.CONFIG.get("logging", "level")) <= logging.DEBUG: + pywps_config.CONFIG.set("logging", "level", "INFO") + # update metadata + for setting_name, setting_value in settings.items(): + if setting_name.startswith("weaver.wps_metadata"): + pywps_setting = setting_name.replace("weaver.wps_metadata_", "") + pywps_config.CONFIG.set("metadata:main", pywps_setting, setting_value) + # add weaver configuration keyword if not already provided + wps_keywords = pywps_config.CONFIG.get("metadata:main", "identification_keywords") + weaver_mode = get_weaver_configuration(settings) + if weaver_mode not in wps_keywords: + wps_keywords += ("," if wps_keywords else "") + weaver_mode + pywps_config.CONFIG.set("metadata:main", "identification_keywords", wps_keywords) # add additional config passed as dictionary of {'section.key': 'value'} if isinstance(config, dict): diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index 44c9acfec..2e8a0fb75 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -38,7 +38,7 @@ LOGGER = get_task_logger(__name__) -def check_status(url=None, response=None, sleep_secs=2, verify=False, settings=None): +def check_status(url=None, response=None, sleep_secs=2, verify=True, settings=None): # type: (Optional[AnyStr], Optional[etree.ElementBase], int, bool, Optional[AnySettingsContainer]) -> WPSExecution """ Run :func:`owslib.wps.WPSExecution.checkStatus` with additional exception handling. diff --git a/weaver/wps_restapi/processes/processes.py b/weaver/wps_restapi/processes/processes.py index c1cca5125..a44abee7a 100644 --- a/weaver/wps_restapi/processes/processes.py +++ b/weaver/wps_restapi/processes/processes.py @@ -49,6 +49,7 @@ get_any_value, get_cookie_headers, get_settings, + get_ssl_verify_option, raise_on_xml_exception, request_extra, wait_secs @@ -89,7 +90,6 @@ def execute_process(self, job_id, url, headers=None, notification_email=None): settings = get_settings(app) task_logger = get_task_logger(__name__) load_pywps_cfg(settings) - ssl_verify = asbool(settings.get("weaver.ssl_verify", True)) wps_out_dir = get_wps_output_dir(settings) task_logger.debug("Job task setup.") @@ -104,6 +104,7 @@ def execute_process(self, job_id, url, headers=None, notification_email=None): try: job.progress = JOB_PROGRESS_DESCRIBE job.save_log(logger=task_logger, message="Execute WPS request for process [{!s}]".format(job.process)) + ssl_verify = get_ssl_verify_option("get", url, settings=settings) wps = WebProcessingService(url=url, headers=get_cookie_headers(headers), verify=ssl_verify) set_wps_language(wps, accept_language=job.accept_language) raise_on_xml_exception(wps._capabilities) # noqa @@ -190,7 +191,7 @@ def execute_process(self, job_id, url, headers=None, notification_email=None): # WPS execution logs can be inserted within the current job log and appear continuously. # Only update internal job fields in case they get referenced elsewhere. job.progress = JOB_PROGRESS_EXECUTE_MONITOR_LOOP - execution = check_status(url=wps_status_path, verify=ssl_verify, sleep_secs=wait_secs(run_step)) + execution = check_status(url=wps_status_path, settings=settings, sleep_secs=wait_secs(run_step)) job_msg = (execution.statusMessage or "").strip() job.response = etree.tostring(execution.response) job.status = map_status(execution.getStatus()) @@ -375,7 +376,7 @@ def submit_job_handler(request, service_url, is_workflow=False, visibility=None) result = execute_process.delay( job_id=job.id, url=clean_ows_url(service_url), - # Convert EnvironHeaders to a simple dict (should cherrypick the required headers) + # Convert EnvironHeaders to a simple dict (should cherry-pick the required headers) headers={k: v for k, v in request.headers.items()}, notification_email=notification_email) LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index b3210755f..5a6ab4f2a 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -1161,6 +1161,7 @@ class ExecutionUnitList(SequenceSchema): class ProcessOffering(MappingSchema): process = Process() processVersion = SchemaNode(String(), missing=drop) + processEndpointWPS1 = SchemaNode(String(), missing=drop, format=URL) jobControlOptions = JobControlOptionsList(missing=drop) outputTransmission = TransmissionModeList(missing=drop)