From 9f630b34c900901382729a3ff2a048b0f364ab25 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 6 Mar 2020 19:43:28 -0500 Subject: [PATCH] support json/xml according to accept header + add job status links (#58) + extra job routes (#86) + compliance with ogc api --- CHANGES.rst | 4 +- config/weaver.ini.example | 6 + requirements.txt | 3 + tests/functional/test_builtin.py | 2 +- tests/wps_restapi/test_api.py | 18 +- tests/wps_restapi/test_jobs.py | 63 +++--- tests/wps_restapi/test_processes.py | 2 +- tests/wps_restapi/test_status_codes.py | 25 +-- weaver/__init__.py | 4 +- weaver/datatype.py | 21 +- weaver/formats.py | 5 + weaver/owsexceptions.py | 65 ++++-- weaver/processes/wps3_process.py | 20 +- weaver/processes/wps_package.py | 66 +----- weaver/tweens.py | 11 +- weaver/utils.py | 57 ++++- weaver/wps_restapi/__init__.py | 8 +- weaver/wps_restapi/api.py | 10 +- weaver/wps_restapi/jobs/__init__.py | 62 ++++-- weaver/wps_restapi/jobs/jobs.py | 127 +++++++---- weaver/wps_restapi/processes/__init__.py | 2 +- weaver/wps_restapi/processes/processes.py | 29 ++- weaver/wps_restapi/swagger_definitions.py | 254 +++++++++++----------- 23 files changed, 492 insertions(+), 372 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f0edcb860..79b6bc55b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -97,14 +97,12 @@ Changes: Changes: -------- -- Add wps languages for other wps requests types: ``DescribeProcess`` and ``GetCapabilities``. +- Add `WPS` languages for other wps requests types: ``DescribeProcess`` and ``GetCapabilities``. Fixes: ------ - Fix a bug where the validation of ``OneOf`` items was casting the value to the first valid possibility. - Now, it doesn't change the value if it's valid without casting it (and still casts it if it's - necessary to make it valid). `1.1.0 `_ (2020-02-17) ======================================================================== diff --git a/config/weaver.ini.example b/config/weaver.ini.example index 4820ba7c0..da7fdec05 100644 --- a/config/weaver.ini.example +++ b/config/weaver.ini.example @@ -17,6 +17,12 @@ mongodb.host = localhost mongodb.port = 27017 mongodb.db_name = weaver +# caching +cache.regions = result +cache.type = memory +cache.result.expire = 3600 +cache.result.enabled = false + # NOTE: # For all below parameters, settings suffixed by `_url` are automatically generated from their corresponding `_path` # settings using `weaver.url` if they are not provided. Otherwise, the explicit definition provided by `_url` suffixed diff --git a/requirements.txt b/requirements.txt index 467885ac5..6913aac6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ alembic argcomplete backports.tempfile; python_version < "3" +beaker celery cffi colander @@ -26,8 +27,10 @@ owslib>=0.19.2; python_version >= "3" owslib>0.19.0; python_version >= "3.8" pymongo pyramid>=1.7.3 +pyramid_beaker>=0.8 pyramid_celery pyramid_mako +pyramid_rewrite pytz pywps pyyaml>=4.2b4 diff --git a/tests/functional/test_builtin.py b/tests/functional/test_builtin.py index d9964b1df..e3fc7b1ac 100644 --- a/tests/functional/test_builtin.py +++ b/tests/functional/test_builtin.py @@ -108,7 +108,7 @@ def test_jsonarray2netcdf_execute(self): continue assert resp.json["status"] == STATUS_SUCCEEDED, \ "Process execution failed. Response body:\n{}".format(resp.json) - resp = self.app.get("{}/result".format(job_url), headers=self.json_headers) + resp = self.app.get("{}/outputs".format(job_url), headers=self.json_headers) assert resp.status_code == 200 assert resp.json["outputs"][0]["id"] == "output" nc_path = resp.json["outputs"][0]["href"] diff --git a/tests/wps_restapi/test_api.py b/tests/wps_restapi/test_api.py index 6a3350369..3e3956e82 100644 --- a/tests/wps_restapi/test_api.py +++ b/tests/wps_restapi/test_api.py @@ -17,7 +17,7 @@ def setUpClass(cls): cls.json_headers = {"Accept": CONTENT_TYPE_APP_JSON, "Content-Type": CONTENT_TYPE_APP_JSON} def test_frontpage_format(self): - resp = self.testapp.get(sd.api_frontpage_uri, headers=self.json_headers) + resp = self.testapp.get(sd.api_frontpage_service.path, headers=self.json_headers) assert resp.status_code == 200 try: sd.FrontpageSchema().deserialize(resp.json) @@ -25,7 +25,7 @@ def test_frontpage_format(self): self.fail("expected valid response format as defined in schema [{!s}]".format(ex)) def test_version_format(self): - resp = self.testapp.get(sd.api_versions_uri, headers=self.json_headers) + resp = self.testapp.get(sd.api_versions_service.path, headers=self.json_headers) assert resp.status_code == 200 try: sd.VersionsSchema().deserialize(resp.json) @@ -33,7 +33,7 @@ def test_version_format(self): self.fail("expected valid response format as defined in schema [{!s}]".format(ex)) def test_conformance_format(self): - resp = self.testapp.get(sd.api_conformance_uri, headers=self.json_headers) + resp = self.testapp.get(sd.api_conformance_service.path, headers=self.json_headers) assert resp.status_code == 200 try: sd.ConformanceSchema().deserialize(resp.json) @@ -41,11 +41,11 @@ def test_conformance_format(self): self.fail("expected valid response format as defined in schema [{!s}]".format(ex)) def test_swagger_api_format(self): - resp = self.testapp.get(sd.api_swagger_ui_uri) + resp = self.testapp.get(sd.api_swagger_ui_service.path) assert resp.status_code == 200 assert "{}".format(sd.API_TITLE) in resp.text - resp = self.testapp.get(sd.api_swagger_json_uri, headers=self.json_headers) + resp = self.testapp.get(sd.api_swagger_json_service.path, headers=self.json_headers) assert resp.status_code == 200 assert "tags" in resp.json assert "info" in resp.json @@ -60,10 +60,10 @@ def test_status_unauthorized_and_forbidden(self): Shouldn't be the default behaviour to employ 403 on both cases. """ with mock.patch("weaver.wps_restapi.api.get_weaver_url", side_effect=HTTPUnauthorized): - resp = self.testapp.get(sd.api_frontpage_uri, headers=self.json_headers, expect_errors=True) + resp = self.testapp.get(sd.api_frontpage_service.path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 401 with mock.patch("weaver.wps_restapi.api.get_weaver_url", side_effect=HTTPForbidden): - resp = self.testapp.get(sd.api_frontpage_uri, headers=self.json_headers, expect_errors=True) + resp = self.testapp.get(sd.api_frontpage_service.path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 403 def test_status_not_found_and_method_not_allowed(self): @@ -75,7 +75,7 @@ def test_status_not_found_and_method_not_allowed(self): assert resp.status_code == 404 # test an existing route with wrong method, shouldn't be the default '404' on both cases - resp = self.testapp.post(sd.api_frontpage_uri, headers=self.json_headers, expect_errors=True) + resp = self.testapp.post(sd.api_frontpage_service.path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 405 @@ -139,7 +139,7 @@ def test_swagger_api_request_base_path_original(self): resp = testapp.get(sd.api_swagger_json_service.path, headers=self.json_headers) assert resp.status_code == 200, "API definition should be accessed directly" assert resp.json["host"] in [self.app_host, "{}:80".format(self.app_host)] - assert resp.json["basePath"] == sd.api_frontpage_uri + assert resp.json["basePath"] == sd.api_frontpage_service.path resp = testapp.get(sd.api_swagger_ui_service.path) assert resp.status_code == 200, "API definition should be accessed directly" diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index e6dbcaf50..33efaa131 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -32,7 +32,7 @@ ) from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC from weaver.warning import TimeZoneInfoAlreadySetWarning -from weaver.wps_restapi.swagger_definitions import jobs_full_uri, jobs_short_uri, process_jobs_uri +from weaver.wps_restapi import swagger_definitions as sd if TYPE_CHECKING: # pylint: disable=W0611,unused-import @@ -161,12 +161,16 @@ def check_job_format(job): assert "status" in job and isinstance(job["status"], six.string_types) assert "message" in job and isinstance(job["message"], six.string_types) assert "percentCompleted" in job and isinstance(job["percentCompleted"], int) - assert "logs" in job and isinstance(job["logs"], six.string_types) + assert "links" in job and isinstance(job["links"], list) and len(job["links"]) + assert all(isinstance(link_info, dict) for link_info in job["links"]) + assert all(any(link_info["rel"] == rel for link_info in job["links"]) for rel in ["self", "logs"]) + for link_info in job["links"]: + assert "href" in link_info and isinstance(link_info["href"], six.string_types) assert job["status"] in JOB_STATUS_VALUES if job["status"] == STATUS_SUCCEEDED: - assert "result" in job and isinstance(job["result"], six.string_types) + assert len([link for link in job["links"] if link["rel"] == "results"]) elif job["status"] == STATUS_FAILED: - assert "exceptions" in job and isinstance(job["exceptions"], six.string_types) + assert len([link for link in job["links"] if link["rel"] == "exceptions"]) @staticmethod def check_basic_jobs_info(response): @@ -206,13 +210,13 @@ def add_params(path, **kwargs): return path + "?" + "&".join("{}={}".format(k, v) for k, v in kwargs.items()) def test_get_jobs_normal_paged(self): - resp = self.app.get(jobs_short_uri, headers=self.json_headers) + resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) self.check_basic_jobs_info(resp) for job_id in resp.json["jobs"]: assert isinstance(job_id, six.string_types) for detail in ("false", 0, "False", "no", "None", "null", None, ""): - path = self.add_params(jobs_short_uri, detail=detail) + path = self.add_params(sd.jobs_service.path, detail=detail) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) for job_id in resp.json["jobs"]: @@ -220,7 +224,7 @@ def test_get_jobs_normal_paged(self): def test_get_jobs_detail_paged(self): for detail in ("true", 1, "True", "yes"): - path = self.add_params(jobs_short_uri, detail=detail) + path = self.add_params(sd.jobs_service.path, detail=detail) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) for job in resp.json["jobs"]: @@ -229,7 +233,7 @@ def test_get_jobs_detail_paged(self): def test_get_jobs_normal_grouped(self): for detail in ("false", 0, "False", "no"): groups = ["process", "service"] - path = self.add_params(jobs_short_uri, detail=detail, groups=",".join(groups)) + path = self.add_params(sd.jobs_service.path, detail=detail, groups=",".join(groups)) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups=groups) for grouped_jobs in resp.json["groups"]: @@ -239,7 +243,7 @@ def test_get_jobs_normal_grouped(self): def test_get_jobs_detail_grouped(self): for detail in ("true", 1, "True", "yes"): groups = ["process", "service"] - path = self.add_params(jobs_short_uri, detail=detail, groups=",".join(groups)) + path = self.add_params(sd.jobs_service.path, detail=detail, groups=",".join(groups)) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups=groups) for grouped_jobs in resp.json["groups"]: @@ -247,7 +251,7 @@ def test_get_jobs_detail_grouped(self): self.check_job_format(job) def test_get_jobs_valid_grouping_by_process(self): - path = self.add_params(jobs_short_uri, detail="false", groups="process") + path = self.add_params(sd.jobs_service.path, detail="false", groups="process") resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups="process") @@ -274,7 +278,7 @@ def test_get_jobs_valid_grouping_by_process(self): pytest.fail("Unknown job grouping 'process' value not expected.") def test_get_jobs_valid_grouping_by_service(self): - path = self.add_params(jobs_short_uri, detail="false", groups="service") + path = self.add_params(sd.jobs_service.path, detail="false", groups="service") resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups="service") @@ -324,7 +328,7 @@ def test_get_jobs_by_encrypted_email(self): assert job.notification_email != email and job.notification_email is not None assert int(job.notification_email, 16) != 0 # email should be encrypted with hex string - path = self.add_params(jobs_short_uri, detail="true", notification_email=email) + path = self.add_params(sd.jobs_service.path, detail="true", notification_email=email) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON @@ -332,14 +336,14 @@ def test_get_jobs_by_encrypted_email(self): assert resp.json["jobs"][0]["jobID"] == job_id def test_get_jobs_process_in_query_normal(self): - path = self.add_params(jobs_short_uri, process=self.job_info[0].process) + path = self.add_params(sd.jobs_service.path, process=self.job_info[0].process) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) assert self.job_info[0].id in resp.json["jobs"], self.message_with_jobs_mapping("expected in") assert self.job_info[1].id not in resp.json["jobs"], self.message_with_jobs_mapping("expected not in") def test_get_jobs_process_in_query_detail(self): - path = self.add_params(jobs_short_uri, process=self.job_info[0].process, detail="true") + path = self.add_params(sd.jobs_service.path, process=self.job_info[0].process, detail="true") resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) job_ids = [j["jobID"] for j in resp.json["jobs"]] @@ -347,14 +351,14 @@ def test_get_jobs_process_in_query_detail(self): assert self.job_info[1].id not in job_ids, self.message_with_jobs_mapping("expected not in") def test_get_jobs_process_in_path_normal(self): - path = process_jobs_uri.format(process_id=self.job_info[0].process) + path = sd.process_jobs_service.path.format(process_id=self.job_info[0].process) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) assert self.job_info[0].id in resp.json["jobs"], self.message_with_jobs_mapping("expected in") assert self.job_info[1].id not in resp.json["jobs"], self.message_with_jobs_mapping("expected not in") def test_get_jobs_process_in_path_detail(self): - path = process_jobs_uri.format(process_id=self.job_info[0].process) + "?detail=true" + path = sd.process_jobs_service.path.format(process_id=self.job_info[0].process) + "?detail=true" resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) job_ids = [j["jobID"] for j in resp.json["jobs"]] @@ -362,49 +366,50 @@ def test_get_jobs_process_in_path_detail(self): assert self.job_info[1].id not in job_ids, self.message_with_jobs_mapping("expected not in") def test_get_jobs_process_unknown_in_path(self): - path = process_jobs_uri.format(process_id="unknown-process-id") + path = sd.process_jobs_service.path.format(process_id="unknown-process-id") resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_process_unknown_in_query(self): - path = self.add_params(jobs_short_uri, process="unknown-process-id") + path = self.add_params(sd.jobs_service.path, process="unknown-process-id") resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_private_process_unauthorized_in_path(self): - path = process_jobs_uri.format(process_id=self.process_private.identifier) + path = sd.process_jobs_service.path.format(process_id=self.process_private.identifier) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 401 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_private_process_not_returned_in_query(self): - path = self.add_params(jobs_short_uri, process=self.process_private.identifier) + path = self.add_params(sd.jobs_service.path, process=self.process_private.identifier) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 401 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_service_and_process_unknown_in_path(self): - path = jobs_full_uri.format(provider_id="unknown-service-id", process_id="unknown-process-id") + path = sd.provider_jobs_service.path.format(provider_id="unknown-service-id", process_id="unknown-process-id") resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_service_and_process_unknown_in_query(self): - path = self.add_params(jobs_short_uri, service="unknown-service-id", process="unknown-process-id") + path = self.add_params(sd.jobs_service.path, service="unknown-service-id", process="unknown-process-id") resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_private_service_public_process_unauthorized_in_path(self): - path = jobs_full_uri.format(provider_id=self.service_private.name, process_id=self.process_public.identifier) + path = sd.provider_jobs_service.path.format(provider_id=self.service_private.name, + process_id=self.process_public.identifier) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 401 assert resp.content_type == CONTENT_TYPE_APP_JSON def test_get_jobs_private_service_public_process_unauthorized_in_query(self): - path = self.add_params(jobs_short_uri, + path = self.add_params(sd.jobs_service.path, service=self.service_private.name, process=self.process_public.identifier) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) @@ -417,7 +422,7 @@ def test_get_jobs_public_service_private_process_unauthorized_in_query(self): it is up to the remote service to hide private processes if the process is visible, the a job can be executed and it is automatically considered public """ - path = self.add_params(jobs_short_uri, + path = self.add_params(sd.jobs_service.path, service=self.service_public.name, process=self.process_private.identifier) with contextlib.ExitStack() as stack: @@ -433,7 +438,7 @@ def test_get_jobs_public_service_no_processes(self): it is up to the remote service to hide private processes if the process is invisible, no job should have been executed nor can be fetched """ - path = self.add_params(jobs_short_uri, + path = self.add_params(sd.jobs_service.path, service=self.service_public.name, process=self.process_private.identifier) with contextlib.ExitStack() as stack: @@ -445,9 +450,9 @@ def test_get_jobs_public_service_no_processes(self): def test_get_jobs_public_with_access_and_request_user(self): """Verifies that corresponding processes are returned when proper access/user-id are respected.""" - uri_direct_jobs = jobs_short_uri - uri_process_jobs = process_jobs_uri.format(process_id=self.process_public.identifier) - uri_provider_jobs = jobs_full_uri.format( + uri_direct_jobs = sd.jobs_service.path + uri_process_jobs = sd.process_jobs_service.path.format(process_id=self.process_public.identifier) + uri_provider_jobs = sd.provider_jobs_service.path.format( provider_id=self.service_public.name, process_id=self.process_public.identifier) admin_public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, self.job_info)) diff --git a/tests/wps_restapi/test_processes.py b/tests/wps_restapi/test_processes.py index b6883c18b..3c2e8ff0c 100644 --- a/tests/wps_restapi/test_processes.py +++ b/tests/wps_restapi/test_processes.py @@ -563,7 +563,7 @@ def test_execute_process_dont_cast_one_of(self): resp = self.app.post_json(path, params=data_execute, headers=self.json_headers) assert resp.status_code == 201, "Expected job submission without inputs created without error." job = self.job_store.fetch_by_id(resp.json["jobID"]) - assert job.inputs[0]["data"] == "100" # not cast to float or integer + assert job.inputs[0]["value"] == "100" # not cast to float or integer def test_execute_process_no_error_not_required_params(self): """ diff --git a/tests/wps_restapi/test_status_codes.py b/tests/wps_restapi/test_status_codes.py index f17d036f3..89d155d03 100644 --- a/tests/wps_restapi/test_status_codes.py +++ b/tests/wps_restapi/test_status_codes.py @@ -4,28 +4,21 @@ from tests.utils import get_test_weaver_app, setup_config_with_mongodb from weaver.formats import CONTENT_TYPE_APP_JSON -from weaver.wps_restapi.swagger_definitions import ( - api_frontpage_uri, - api_swagger_json_uri, - api_swagger_ui_uri, - api_versions_uri, - jobs_full_uri, - jobs_short_uri -) +from weaver.wps_restapi import swagger_definitions as sd TEST_PUBLIC_ROUTES = [ - api_frontpage_uri, - api_swagger_ui_uri, - api_swagger_json_uri, - api_versions_uri, + sd.api_frontpage_service.path, + sd.api_swagger_ui_service.path, + sd.api_swagger_json_service.path, + sd.api_versions_service.path, ] TEST_FORBIDDEN_ROUTES = [ - jobs_short_uri, # should always be visible - jobs_full_uri, # could be 401 + sd.jobs_service.path, # should always be visible + sd.provider_jobs_service.path, # could be 401 ] TEST_NOTFOUND_ROUTES = [ - "/jobs/not-found", - "/providers/not-found", + sd.job_service.path.format(job_id="not-found"), + sd.provider_service.path.format(provider_id="not-found"), ] diff --git a/weaver/__init__.py b/weaver/__init__.py index 1ff88c21d..0c9deebe2 100644 --- a/weaver/__init__.py +++ b/weaver/__init__.py @@ -40,6 +40,7 @@ def main(global_config, **settings): from weaver.processes.utils import register_wps_processes_from_config from weaver.utils import parse_extra_options, get_settings from pyramid.config import Configurator + from pyramid_beaker import set_cache_regions_from_settings # validate and fix configuration weaver_config = get_weaver_configuration(settings) @@ -49,11 +50,12 @@ def main(global_config, **settings): settings.update(parse_extra_options(settings.get("weaver.extra_options", ""))) local_config = Configurator(settings=settings) + set_cache_regions_from_settings(settings) if global_config.get("__file__") is not None: local_config.include("pyramid_celery") local_config.configure_celery(global_config["__file__"]) - + local_config.include("pyramid_beaker") local_config.include("weaver") LOGGER.info("Registering builtin processes...") diff --git a/weaver/datatype.py b/weaver/datatype.py index 64f1736a5..9da70e843 100644 --- a/weaver/datatype.py +++ b/weaver/datatype.py @@ -13,6 +13,7 @@ from pywps import Process as ProcessWPS from weaver.exceptions import ProcessInstanceError +from weaver.formats import CONTENT_TYPE_APP_JSON, LANGUAGE_EN_US from weaver.processes.types import PROCESS_APPLICATION, PROCESS_BUILTIN, PROCESS_TEST, PROCESS_WORKFLOW, PROCESS_WPS from weaver.status import ( JOB_STATUS_CATEGORIES, @@ -532,8 +533,8 @@ def response(self, response): def _job_url(self, settings): base_job_url = get_wps_restapi_base_url(settings) if self.service is not None: - base_job_url += sd.provider_uri.format(provider_id=self.service) - job_path = sd.process_job_uri.format(process_id=self.process, job_id=self.id) + base_job_url += sd.provider_service.path.format(provider_id=self.service) + job_path = sd.process_job_service.path.format(process_id=self.process, job_id=self.id) return "{base_job_url}{job_path}".format(base_job_url=base_job_url, job_path=job_path) def json(self, container=None): # pylint: disable=W0221,arguments-differ @@ -551,17 +552,23 @@ def json(self, container=None): # pylint: disable=W0221,arguments-differ "message": self.status_message, "duration": self.duration_str, "percentCompleted": self.progress, + "links": [] } job_url = self._job_url(settings) - # TODO: use links (https://github.com/crim-ca/weaver/issues/58) + job_json["links"].append({"href": job_url, "rel": "self", "title": "Job status."}) + job_links = ["logs", "inputs"] if self.status in JOB_STATUS_CATEGORIES[STATUS_CATEGORY_FINISHED]: job_status = map_status(self.status) if job_status == STATUS_SUCCEEDED: - resource_type = "result" + job_links.extend(["outputs", "results"]) else: - resource_type = "exceptions" - job_json[resource_type] = "{job_url}/{res}".format(job_url=job_url, res=resource_type.lower()) - job_json["logs"] = "{job_url}/logs".format(job_url=job_url) + job_links.extend(["exceptions"]) + for link_type in job_links: + link_href = "{job_url}/{res}".format(job_url=job_url, res=link_type) + job_json["links"].append({"href": link_href, "rel": link_type, "title": "Job {}.".format(link_type)}) + link_meta = {"type": CONTENT_TYPE_APP_JSON, "hreflang": LANGUAGE_EN_US} + for link in job_json["links"]: + link.update(link_meta) return sd.JobStatusInfo().deserialize(job_json) def params(self): diff --git a/weaver/formats.py b/weaver/formats.py index 1cb078dbb..046fb6deb 100644 --- a/weaver/formats.py +++ b/weaver/formats.py @@ -13,6 +13,11 @@ from weaver.typedefs import JSON # noqa: F401 from typing import AnyStr, Dict, Tuple, Union # noqa: F401 +# Languages +LANGUAGE_EN_CA = "en-CA" +LANGUAGE_FR_CA = "fr-CA" +LANGUAGE_EN_US = "en-US" + # Content-Types CONTENT_TYPE_APP_FORM = "application/x-www-form-urlencoded" CONTENT_TYPE_APP_NETCDF = "application/x-netcdf" diff --git a/weaver/owsexceptions.py b/weaver/owsexceptions.py index 12d9fbc77..e2812bb70 100644 --- a/weaver/owsexceptions.py +++ b/weaver/owsexceptions.py @@ -26,7 +26,12 @@ from webob.acceptparse import create_accept_header from zope.interface import implementer -from weaver.formats import CONTENT_TYPE_APP_JSON, CONTENT_TYPE_TEXT_XML +from weaver.formats import ( + CONTENT_TYPE_APP_XML, + CONTENT_TYPE_APP_JSON, + CONTENT_TYPE_TEXT_HTML, + CONTENT_TYPE_TEXT_XML +) from weaver.utils import clean_json_text_body from weaver.warning import MissingParameterWarning, UnsupportedOperationWarning @@ -40,7 +45,7 @@ class OWSException(Response, Exception): code = "NoApplicableCode" value = None locator = "NoApplicableCode" - explanation = "Unknown Error" + description = "Unknown Error" page_template = Template("""\ @@ -66,10 +71,12 @@ def __init__(self, detail=None, value=None, **kw): status = status.status elif not status: status = HTTPOk().status + self.code = str(kw.pop("code", self.code)) + self.description = str(detail or kw.pop("description", self.description)) Response.__init__(self, status=status, **kw) Exception.__init__(self, detail) - self.message = detail or self.explanation - self.content_type = CONTENT_TYPE_TEXT_XML + self.message = detail or self.description + self.content_type = CONTENT_TYPE_APP_JSON if value: self.locator = value @@ -84,16 +91,26 @@ def __repr__(self): @staticmethod def json_formatter(status, body, title, environ): # noqa: F811 # type: (AnyStr, AnyStr, AnyStr, SettingsType) -> JSON - body = clean_json_text_body(body) - return {"description": body, "code": int(status.split()[0]), "status": status, "title": title} + body = clean_json_text_body(body) # message/description + code = int(status.split()[0]) # HTTP status code + body = {"description": body, "code": title} # title is the string OGC 'code' + if code >= 400: + body["error"] = {"code": code, "status": status} + return body def prepare(self, environ): if not self.body: accept_value = environ.get("HTTP_ACCEPT", "") accept = create_accept_header(accept_value) - # Attempt to match xml or json, if those don't match, we will fall through to defaulting to xml - match = accept.best_match([CONTENT_TYPE_TEXT_XML, CONTENT_TYPE_APP_JSON]) + # Attempt to match XML or JSON, if those don't match, we will fall back to defaulting to JSON + # since browsers add HTML automatically and it is closer to XML, we 'allow' it only to catch this + # explicit case and fallback to JSON manually + match = accept.best_match([CONTENT_TYPE_TEXT_HTML, CONTENT_TYPE_APP_JSON, + CONTENT_TYPE_TEXT_XML, CONTENT_TYPE_APP_XML], + default_match=CONTENT_TYPE_APP_JSON) + if match == CONTENT_TYPE_TEXT_HTML: + match = CONTENT_TYPE_APP_JSON if match == CONTENT_TYPE_APP_JSON: self.content_type = CONTENT_TYPE_APP_JSON @@ -106,21 +123,21 @@ class JsonPageTemplate(object): def __init__(self, excobj): self.excobj = excobj - def substitute(self, code, locator, message): # noqa: W0613 + def substitute(self, code, locator, message): # noqa: E811 + msg_code = getattr(self.excobj, "code", None) return json.dumps(self.excobj.json_formatter( - status=self.excobj.status, body=message, title=None, environ=environ)) + status=self.excobj.status, body=message, title=msg_code, environ=environ)) page_template = JsonPageTemplate(self) - + args = {"code": self.code, "locator": self.locator, "message": self.message} else: self.content_type = CONTENT_TYPE_TEXT_XML page_template = self.page_template - - args = { - "code": _html_escape(self.code), - "locator": _html_escape(self.locator), - "message": _html_escape(self.message or ""), - } + args = { + "code": _html_escape(self.code), + "locator": _html_escape(self.locator), + "message": _html_escape(self.message or ""), + } page = page_template.substitute(**args) if isinstance(page, text_type): page = page.encode(self.charset if self.charset else "UTF-8") @@ -148,7 +165,7 @@ def __call__(self, environ, start_response): class OWSAccessForbidden(OWSException): locator = "AccessUnauthorized" - explanation = "Access to this service is unauthorized." + description = "Access to this service is unauthorized." def __init__(self, *args, **kwargs): kwargs["status"] = HTTPUnauthorized @@ -157,7 +174,7 @@ def __init__(self, *args, **kwargs): class OWSNotFound(OWSException): locator = "NotFound" - explanation = "This resource does not exist." + description = "This resource does not exist." def __init__(self, *args, **kwargs): kwargs["status"] = HTTPNotFound @@ -166,7 +183,7 @@ def __init__(self, *args, **kwargs): class OWSNotAcceptable(OWSException): locator = "NotAcceptable" - explanation = "Access to this service failed." + description = "Access to this service failed." def __init__(self, *args, **kwargs): kwargs["status"] = HTTPNotAcceptable @@ -177,7 +194,7 @@ class OWSNoApplicableCode(OWSException): """WPS Bad Request Exception""" code = "NoApplicableCode" locator = "" - explanation = "Parameter value is missing" + description = "Parameter value is missing" def __init__(self, *args, **kwargs): kwargs["status"] = HTTPBadRequest @@ -189,7 +206,7 @@ class OWSMissingParameterValue(OWSException): """MissingParameterValue WPS Exception""" code = "MissingParameterValue" locator = "" - explanation = "Parameter value is missing" + description = "Parameter value is missing" def __init__(self, *args, **kwargs): kwargs["status"] = HTTPBadRequest @@ -201,7 +218,7 @@ class OWSInvalidParameterValue(OWSException): """InvalidParameterValue WPS Exception""" code = "InvalidParameterValue" locator = "" - explanation = "Parameter value is not acceptable." + description = "Parameter value is not acceptable." def __init__(self, *args, **kwargs): kwargs["status"] = HTTPNotAcceptable @@ -212,7 +229,7 @@ def __init__(self, *args, **kwargs): class OWSNotImplemented(OWSException): code = "NotImplemented" locator = "" - explanation = "Operation is not implemented." + description = "Operation is not implemented." def __init__(self, *args, **kwargs): kwargs["status"] = HTTPNotImplemented diff --git a/weaver/processes/wps3_process.py b/weaver/processes/wps3_process.py index 3e605938d..1528ccbfc 100644 --- a/weaver/processes/wps3_process.py +++ b/weaver/processes/wps3_process.py @@ -33,13 +33,7 @@ ) from weaver.visibility import VISIBILITY_PUBLIC from weaver.warning import MissingParameterWarning -from weaver.wps_restapi.swagger_definitions import ( - process_jobs_uri, - process_results_uri, - process_uri, - process_visibility_uri, - processes_uri -) +from weaver.wps_restapi import swagger_definitions as sd if TYPE_CHECKING: from weaver.typedefs import JSON, UpdateStatusPartialFunction # noqa: F401 @@ -153,7 +147,7 @@ def is_visible(self): """ LOGGER.debug("Get process WPS visibility request for [%s]", self.process) response = self.make_request(method="GET", - url=self.url + process_visibility_uri.format(process_id=self.process), + url=self.url + sd.process_visibility_service.path.format(process_id=self.process), retry=False, status_code_mock=HTTPUnauthorized.code) if response.status_code in (HTTPUnauthorized.code, HTTPForbidden.code): @@ -171,7 +165,7 @@ def is_visible(self): def set_visibility(self, visibility): self.update_status("Updating process visibility on remote ADES.", REMOTE_JOB_PROGRESS_VISIBLE, status.STATUS_RUNNING) - path = self.url + process_visibility_uri.format(process_id=self.process) + path = self.url + sd.process_visibility_service.path.format(process_id=self.process) user_headers = deepcopy(self.headers) user_headers.update(self.get_user_auth_header()) @@ -184,7 +178,7 @@ def set_visibility(self, visibility): response.raise_for_status() def describe_process(self): - path = self.url + process_uri.format(process_id=self.process) + path = self.url + sd.process_service.path.format(process_id=self.process) LOGGER.debug("Describe process WPS request for [%s] at [%s]", self.process, path) response = self.make_request(method="GET", url=path, @@ -206,7 +200,7 @@ def describe_process(self): def deploy(self): self.update_status("Deploying process on remote ADES.", REMOTE_JOB_PROGRESS_DEPLOY, status.STATUS_RUNNING) - path = self.url + processes_uri + path = self.url + sd.processes_service.path user_headers = deepcopy(self.headers) user_headers.update(self.get_user_auth_header()) @@ -270,7 +264,7 @@ def execute(self, workflow_inputs, out_dir, expected_outputs): response="document", inputs=execute_body_inputs, outputs=execute_body_outputs) - request_url = self.url + process_jobs_uri.format(process_id=self.process) + request_url = self.url + sd.process_jobs_service.path.format(process_id=self.process) response = self.make_request(method="POST", url=request_url, json=execute_body, @@ -349,7 +343,7 @@ def get_job_status(self, job_status_uri, retry=True): return job_status def get_job_results(self, job_id): - result_url = self.url + process_results_uri.format(process_id=self.process, job_id=job_id) + result_url = self.url + sd.process_results_service.path.format(process_id=self.process, job_id=job_id) response = self.make_request(method="GET", url=result_url, retry=True) diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 5cee7edd1..11867ffc1 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -82,19 +82,20 @@ get_settings, get_url_without_query, null, - str2bytes + str2bytes, + transform_json ) from weaver.wps import get_wps_output_dir -from weaver.wps_restapi.swagger_definitions import process_uri +from weaver.wps_restapi.swagger_definitions import process_service if TYPE_CHECKING: # pylint: disable=W0611,unused-import from weaver.datatype import Job # noqa: F401 from weaver.status import AnyStatusType # noqa: F401 from weaver.typedefs import ( # noqa: F401 - ToolPathObjectType, CWLFactoryCallable, CWL, AnyKey, AnyValue as AnyValueType, JSON, XML, Number + ToolPathObjectType, CWLFactoryCallable, CWL, AnyValue as AnyValueType, JSON, XML, Number ) - from typing import Any, AnyStr, Callable, Dict, List, Optional, Tuple, Type, Union # noqa: F401 + from typing import Any, AnyStr, Dict, List, Optional, Tuple, Type, Union # noqa: F401 from cwltool.process import Process as ProcessCWL # noqa: F401 from pywps.app import WPSRequest # noqa: F401 from pywps.response.execute import ExecuteResponse # noqa: F401 @@ -226,7 +227,7 @@ def get_process_location(process_id_or_url, data_source=None): return process_id_or_url data_source_url = retrieve_data_source_url(data_source) process_id = get_sane_name(process_id_or_url) - process_url = process_uri.format(process_id=process_id) + process_url = process_service.path.format(process_id=process_id) return "{host}{path}".format(host=data_source_url, path=process_url) @@ -1152,61 +1153,6 @@ def _merge_package_io(wps_io_list, cwl_io_list, io_select): return updated_io_list -def transform_json(json_data, # type: ANY_IO_Type - rename=None, # type: Optional[Dict[AnyKey, Any]] - remove=None, # type: Optional[List[AnyKey]] - add=None, # type: Optional[Dict[AnyKey, Any]] - replace_values=None, # type: Optional[Dict[AnyKey, Any]] - replace_func=None, # type: Optional[Dict[AnyKey, Callable[[Any], Any]]] - ): # type: (...) -> ANY_IO_Type - """ - Transforms the input json_data with different methods. - The transformations are applied in the same order as the arguments. - """ - rename = rename or {} - remove = remove or [] - add = add or {} - replace_values = replace_values or {} - replace_func = replace_func or {} - - # rename - for k, v in rename.items(): - if k in json_data: - json_data[v] = json_data.pop(k) - - # remove - for r_k in remove: - json_data.pop(r_k, None) - - # add - for k, v in add.items(): - json_data[k] = v - - # replace values - for key, value in json_data.items(): - for old_value, new_value in replace_values.items(): - if value == old_value: - json_data[key] = new_value - - # replace with function call - for k, func in replace_func.items(): - if k in json_data: - json_data[k] = func(json_data[k]) - - # also rename if the type of the value is a list of dicts - for key, value in json_data.items(): - if isinstance(value, list): - for nested_item in value: - if isinstance(nested_item, dict): - for k, v in rename.items(): - if k in nested_item: - nested_item[v] = nested_item.pop(k) - for k, func in replace_func.items(): - if k in nested_item: - nested_item[k] = func(nested_item[k]) - return json_data - - def _merge_package_inputs_outputs(wps_inputs_list, # type: List[ANY_IO_Type] cwl_inputs_list, # type: List[WPS_Input_Type] wps_outputs_list, # type: List[ANY_IO_Type] diff --git a/weaver/tweens.py b/weaver/tweens.py index 81d181ebc..32e9e09a6 100644 --- a/weaver/tweens.py +++ b/weaver/tweens.py @@ -13,6 +13,7 @@ def ows_response_tween(request, handler): + log_level = logging.INFO # real http errors are logged as normal entry try: result = handler(request) if hasattr(handler, OWS_TWEEN_HANDLED): @@ -25,24 +26,26 @@ def ows_response_tween(request, handler): raised_error = err raised_error._json_formatter = OWSException.json_formatter return_error = raised_error - exc_info_err = sys.exc_info() + exc_info_err = False except OWSException as err: LOGGER.debug("direct ows exception response") - LOGGER.exception("Raised exception: [%r]\nReturned exception: [%r]", err, err) raised_error = err return_error = err - exc_info_err = sys.exc_info() + exc_info_err = False except NotImplementedError as err: LOGGER.debug("not implemented error -> ows exception response") raised_error = err return_error = OWSNotImplemented(str(err)) exc_info_err = sys.exc_info() + log_level = logging.WARNING except Exception as err: LOGGER.debug("unhandled %s exception -> ows exception response", type(err).__name__) raised_error = err return_error = OWSException(detail=str(err), status=HTTPInternalServerError) exc_info_err = sys.exc_info() - LOGGER.error("Raised exception: [%r]\nReturned exception: [%r]", raised_error, return_error, exc_info=exc_info_err) + log_level = logging.ERROR + LOGGER.log(log_level, "Raised exception: [%r]\nReturned exception: [%r]", + raised_error, return_error, exc_info=exc_info_err) return return_error diff --git a/weaver/utils.py b/weaver/utils.py index 6ef1554b0..8be35f2de 100644 --- a/weaver/utils.py +++ b/weaver/utils.py @@ -33,7 +33,7 @@ AnyValue, AnyKey, AnySettingsContainer, AnyRegistryContainer, AnyHeadersContainer, AnyResponseType, HeadersType, SettingsType, JSON, XML, Number ) - from typing import Union, Any, Dict, List, AnyStr, Iterable, Optional, Type # noqa: F401 + from typing import Union, Any, Callable, Dict, List, AnyStr, Iterable, Optional, Type # noqa: F401 LOGGER = logging.getLogger(__name__) @@ -590,3 +590,58 @@ def clean_json_text_body(body): body_parts = [p[0].upper() + p[1:] for p in body_parts if len(p)] # capitalize first word body_parts = " ".join(p for p in body_parts if p) return body_parts + + +def transform_json(json_data, # type: JSON + rename=None, # type: Optional[Dict[AnyKey, Any]] + remove=None, # type: Optional[List[AnyKey]] + add=None, # type: Optional[Dict[AnyKey, Any]] + replace_values=None, # type: Optional[Dict[AnyKey, Any]] + replace_func=None, # type: Optional[Dict[AnyKey, Callable[[Any], Any]]] + ): # type: (...) -> JSON + """ + Transforms the input ``json_data`` with different methods. + The transformations are applied in the same order as the arguments. + """ + rename = rename or {} + remove = remove or [] + add = add or {} + replace_values = replace_values or {} + replace_func = replace_func or {} + + # rename + for k, v in rename.items(): + if k in json_data: + json_data[v] = json_data.pop(k) + + # remove + for r_k in remove: + json_data.pop(r_k, None) + + # add + for k, v in add.items(): + json_data[k] = v + + # replace values + for key, value in json_data.items(): + for old_value, new_value in replace_values.items(): + if value == old_value: + json_data[key] = new_value + + # replace with function call + for k, func in replace_func.items(): + if k in json_data: + json_data[k] = func(json_data[k]) + + # also rename if the type of the value is a list of dicts + for key, value in json_data.items(): + if isinstance(value, list): + for nested_item in value: + if isinstance(nested_item, dict): + for k, v in rename.items(): + if k in nested_item: + nested_item[v] = nested_item.pop(k) + for k, func in replace_func.items(): + if k in nested_item: + nested_item[k] = func(nested_item[k]) + return json_data diff --git a/weaver/wps_restapi/__init__.py b/weaver/wps_restapi/__init__.py index 251c9e4ac..8b20e1c3d 100644 --- a/weaver/wps_restapi/__init__.py +++ b/weaver/wps_restapi/__init__.py @@ -21,6 +21,12 @@ def includeme(config): config.include("weaver.wps_restapi.processes") config.include("weaver.wps_restapi.quotation") config.include("pyramid_mako") + config.include("pyramid_rewrite") + # attempt finding a not found route using either an added or removed trailing slash according to situation + config.add_rewrite_rule(r"/(?P.*)/", r"/%(path)s") + config.add_notfound_view(api.not_found_or_method_not_allowed, append_slash=True) + config.add_forbidden_view(api.unauthorized_or_forbidden) + config.add_route(**sd.service_api_route_info(sd.api_frontpage_service, settings)) config.add_route(**sd.service_api_route_info(sd.api_swagger_json_service, settings)) config.add_route(**sd.service_api_route_info(sd.api_swagger_ui_service, settings)) @@ -36,5 +42,3 @@ def includeme(config): request_method="GET", renderer=OUTPUT_FORMAT_JSON) config.add_view(api.api_conformance, route_name=sd.api_conformance_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_notfound_view(api.not_found_or_method_not_allowed, append_slash=True) - config.add_forbidden_view(api.unauthorized_or_forbidden) diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index 8e9bb7b73..0800b6afc 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -51,13 +51,13 @@ def api_frontpage(request): weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None - weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None + weaver_api_def = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None - weaver_conform_url = weaver_url + sd.api_conformance_uri - weaver_process_url = weaver_url + sd.processes_uri + weaver_conform_url = weaver_url + sd.api_conformance_service.path + weaver_process_url = weaver_url + sd.processes_service.path weaver_links = [ {"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"}, {"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON, @@ -145,7 +145,7 @@ def api_swagger_json(request, use_docstring_summary=True): swagger_base_path = weaver_parsed_url.path else: swagger_base_spec["host"] = request.host - swagger_base_path = sd.api_frontpage_uri + swagger_base_path = sd.api_frontpage_service.path swagger.swagger = swagger_base_spec return swagger.generate(title=sd.API_TITLE, version=weaver_version, base_path=swagger_base_path) @@ -154,7 +154,7 @@ def api_swagger_json(request, use_docstring_summary=True): schema=sd.SwaggerUIEndpoint(), response_schemas=sd.get_api_swagger_ui_responses) def api_swagger_ui(request): """weaver REST API swagger-ui schema documentation (this page).""" - json_path = wps_restapi_base_path(request.registry.settings) + sd.api_swagger_json_uri + json_path = wps_restapi_base_path(request.registry.settings) + sd.api_swagger_json_service.path json_path = json_path.lstrip("/") # if path starts by '/', swagger-ui doesn't find it on remote data_mako = {"api_title": sd.API_TITLE, "api_swagger_json_path": json_path} return render_to_response("templates/swagger_ui.mako", data_mako, request=request) diff --git a/weaver/wps_restapi/jobs/__init__.py b/weaver/wps_restapi/jobs/__init__.py index 906c2691d..197d5f46d 100644 --- a/weaver/wps_restapi/jobs/__init__.py +++ b/weaver/wps_restapi/jobs/__init__.py @@ -10,55 +10,73 @@ def includeme(config): LOGGER.info("Adding WPS REST API jobs...") settings = config.registry.settings - config.add_route(**sd.service_api_route_info(sd.jobs_short_service, settings)) - config.add_route(**sd.service_api_route_info(sd.jobs_full_service, settings)) - config.add_route(**sd.service_api_route_info(sd.job_short_service, settings)) - config.add_route(**sd.service_api_route_info(sd.job_full_service, settings)) - config.add_route(**sd.service_api_route_info(sd.results_short_service, settings)) - config.add_route(**sd.service_api_route_info(sd.results_full_service, settings)) - config.add_route(**sd.service_api_route_info(sd.exceptions_short_service, settings)) - config.add_route(**sd.service_api_route_info(sd.exceptions_full_service, settings)) - config.add_route(**sd.service_api_route_info(sd.logs_short_service, settings)) - config.add_route(**sd.service_api_route_info(sd.logs_full_service, settings)) + config.add_route(**sd.service_api_route_info(sd.jobs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.job_service, settings)) + config.add_route(**sd.service_api_route_info(sd.job_results_service, settings)) + config.add_route(**sd.service_api_route_info(sd.job_outputs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.job_inputs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.job_exceptions_service, settings)) + config.add_route(**sd.service_api_route_info(sd.job_logs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_job_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_jobs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_results_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_outputs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_inputs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_exceptions_service, settings)) + config.add_route(**sd.service_api_route_info(sd.provider_logs_service, settings)) config.add_route(**sd.service_api_route_info(sd.process_jobs_service, settings)) config.add_route(**sd.service_api_route_info(sd.process_job_service, settings)) config.add_route(**sd.service_api_route_info(sd.process_results_service, settings)) + config.add_route(**sd.service_api_route_info(sd.process_outputs_service, settings)) + config.add_route(**sd.service_api_route_info(sd.process_inputs_service, settings)) config.add_route(**sd.service_api_route_info(sd.process_exceptions_service, settings)) config.add_route(**sd.service_api_route_info(sd.process_logs_service, settings)) config.add_view(j.get_queried_jobs, route_name=sd.process_jobs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_queried_jobs, route_name=sd.jobs_short_service.name, + config.add_view(j.get_queried_jobs, route_name=sd.jobs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_queried_jobs, route_name=sd.jobs_full_service.name, + config.add_view(j.get_queried_jobs, route_name=sd.provider_jobs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_status, route_name=sd.job_short_service.name, + config.add_view(j.get_job_status, route_name=sd.job_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_status, route_name=sd.job_full_service.name, + config.add_view(j.get_job_status, route_name=sd.provider_job_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) config.add_view(j.get_job_status, route_name=sd.process_job_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.cancel_job, route_name=sd.job_short_service.name, + config.add_view(j.cancel_job, route_name=sd.job_service.name, request_method="DELETE", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.cancel_job, route_name=sd.job_full_service.name, + config.add_view(j.cancel_job, route_name=sd.provider_job_service.name, request_method="DELETE", renderer=OUTPUT_FORMAT_JSON) config.add_view(j.cancel_job, route_name=sd.process_job_service.name, request_method="DELETE", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_results, route_name=sd.results_short_service.name, + config.add_view(j.get_job_results, route_name=sd.job_results_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_results, route_name=sd.results_full_service.name, + config.add_view(j.get_job_results, route_name=sd.provider_results_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) config.add_view(j.get_job_results, route_name=sd.process_results_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_exceptions, route_name=sd.exceptions_short_service.name, + config.add_view(j.get_job_outputs, route_name=sd.job_outputs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_exceptions, route_name=sd.exceptions_full_service.name, + config.add_view(j.get_job_outputs, route_name=sd.provider_outputs_service.name, + request_method="GET", renderer=OUTPUT_FORMAT_JSON) + config.add_view(j.get_job_outputs, route_name=sd.process_outputs_service.name, + request_method="GET", renderer=OUTPUT_FORMAT_JSON) + config.add_view(j.get_job_inputs, route_name=sd.job_inputs_service.name, + request_method="GET", renderer=OUTPUT_FORMAT_JSON) + config.add_view(j.get_job_inputs, route_name=sd.provider_inputs_service.name, + request_method="GET", renderer=OUTPUT_FORMAT_JSON) + config.add_view(j.get_job_inputs, route_name=sd.process_inputs_service.name, + request_method="GET", renderer=OUTPUT_FORMAT_JSON) + config.add_view(j.get_job_exceptions, route_name=sd.job_exceptions_service.name, + request_method="GET", renderer=OUTPUT_FORMAT_JSON) + config.add_view(j.get_job_exceptions, route_name=sd.provider_exceptions_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) config.add_view(j.get_job_exceptions, route_name=sd.process_exceptions_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_logs, route_name=sd.logs_short_service.name, + config.add_view(j.get_job_logs, route_name=sd.job_logs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(j.get_job_logs, route_name=sd.logs_full_service.name, + config.add_view(j.get_job_logs, route_name=sd.provider_logs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) config.add_view(j.get_job_logs, route_name=sd.process_logs_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index b6e66046a..affb96ecb 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -25,6 +25,7 @@ ServiceNotFound, log_unhandled_exceptions ) +from weaver.owsexceptions import OWSNotFound from weaver.store.base import StoreJobs, StoreProcesses, StoreServices from weaver.utils import get_any_id, get_any_value, get_settings, get_url_without_query from weaver.visibility import VISIBILITY_PUBLIC @@ -36,6 +37,7 @@ if TYPE_CHECKING: from weaver.typedefs import AnySettingsContainer, JSON # noqa: F401 from typing import AnyStr, Optional, Tuple, Union # noqa: F401 + from pyramid.httpexceptions import HTTPException # noqa: F401 LOGGER = get_task_logger(__name__) @@ -107,15 +109,21 @@ def get_job(request): try: job = store.fetch_by_id(job_id, request=request) except JobNotFound: - raise HTTPNotFound("Could not find job with specified 'job_id'.") + raise OWSNotFound(code="NoSuchJob", description="Could not find job with specified 'job_id'.") provider_id = request.matchdict.get("provider_id", job.service) process_id = request.matchdict.get("process_id", job.process) if job.service != provider_id: - raise HTTPNotFound("Could not find job with specified 'provider_id'.") + raise OWSNotFound( + code="NoSuchProvider", + description="Could not find job corresponding to specified 'provider_id'." + ) if job.process != process_id: - raise HTTPNotFound("Could not find job with specified 'process_id'.") + raise OWSNotFound( + code="NoSuchProcess", + description="Could not find job corresponding to specified 'process_id'." + ) return job @@ -150,21 +158,30 @@ def validate_service_process(request): if process_name not in [p.id for p in processes]: raise ProcessNotFound except (ServiceNotFound, ProcessNotFound): - raise HTTPNotFound("{} of id '{}' cannot be found.".format(item_type, item_test)) + raise HTTPNotFound(json={ + "code": "NoSuch{}".format(item_type), + "description": "{} of id '{}' cannot be found.".format(item_type, item_test) + }) except (ServiceNotAccessible, ProcessNotAccessible): - raise HTTPUnauthorized("{} of id '{}' is not accessible.".format(item_type, item_test)) + raise HTTPUnauthorized(json={ + "code": "Unauthorized{}".format(item_type), + "description": "{} of id '{}' is not accessible.".format(item_type, item_test) + }) except InvalidIdentifierValue as ex: - raise HTTPBadRequest(str(ex)) + raise HTTPBadRequest(json={ + "code": InvalidIdentifierValue.__name__, + "description": str(ex) + }) return service_name, process_name @sd.process_jobs_service.get(tags=[sd.TAG_PROCESSES, sd.TAG_JOBS], renderer=OUTPUT_FORMAT_JSON, schema=sd.GetProcessJobsEndpoint(), response_schemas=sd.get_all_jobs_responses) -@sd.jobs_full_service.get(tags=[sd.TAG_JOBS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.GetProviderJobsEndpoint(), response_schemas=sd.get_all_jobs_responses) -@sd.jobs_short_service.get(tags=[sd.TAG_JOBS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.GetJobsEndpoint(), response_schemas=sd.get_all_jobs_responses) +@sd.provider_jobs_service.get(tags=[sd.TAG_JOBS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.GetProviderJobsEndpoint(), response_schemas=sd.get_all_jobs_responses) +@sd.jobs_service.get(tags=[sd.TAG_JOBS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.GetJobsEndpoint(), response_schemas=sd.get_all_jobs_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobsResponse.description) def get_queried_jobs(request): """ @@ -208,10 +225,10 @@ def _job_list(jobs): return HTTPOk(json=body) -@sd.job_full_service.get(tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.FullJobEndpoint(), response_schemas=sd.get_single_job_status_responses) -@sd.job_short_service.get(tags=[sd.TAG_JOBS, sd.TAG_STATUS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.ShortJobEndpoint(), response_schemas=sd.get_single_job_status_responses) +@sd.provider_job_service.get(tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.FullJobEndpoint(), response_schemas=sd.get_single_job_status_responses) +@sd.job_service.get(tags=[sd.TAG_JOBS, sd.TAG_STATUS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ShortJobEndpoint(), response_schemas=sd.get_single_job_status_responses) @sd.process_job_service.get(tags=[sd.TAG_PROCESSES, sd.TAG_JOBS, sd.TAG_STATUS], renderer=OUTPUT_FORMAT_JSON, schema=sd.GetProcessJobEndpoint(), response_schemas=sd.get_single_job_status_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobStatusResponse.description) @@ -223,10 +240,10 @@ def get_job_status(request): return HTTPOk(json=job.json(request)) -@sd.job_full_service.delete(tags=[sd.TAG_JOBS, sd.TAG_DISMISS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.FullJobEndpoint(), response_schemas=sd.delete_job_responses) -@sd.job_short_service.delete(tags=[sd.TAG_JOBS, sd.TAG_DISMISS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.ShortJobEndpoint(), response_schemas=sd.delete_job_responses) +@sd.provider_job_service.delete(tags=[sd.TAG_JOBS, sd.TAG_DISMISS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.FullJobEndpoint(), response_schemas=sd.delete_job_responses) +@sd.job_service.delete(tags=[sd.TAG_JOBS, sd.TAG_DISMISS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ShortJobEndpoint(), response_schemas=sd.delete_job_responses) @sd.process_job_service.delete(tags=[sd.TAG_PROCESSES, sd.TAG_JOBS, sd.TAG_DISMISS], renderer=OUTPUT_FORMAT_JSON, schema=sd.DeleteProcessJobEndpoint(), response_schemas=sd.delete_job_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorDeleteJobResponse.description) @@ -251,6 +268,40 @@ def cancel_job(request): }) +@sd.provider_inputs_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ProviderInputsEndpoint(), response_schemas=sd.get_job_inputs_responses) +@sd.process_inputs_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ProcessInputsEndpoint(), response_schemas=sd.get_job_inputs_responses) +@sd.job_inputs_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, + schema=sd.JobInputsEndpoint(), response_schemas=sd.get_job_inputs_responses) +@log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobResultsResponse.description) +def get_job_inputs(request): + # type: (Request) -> HTTPException + """ + Retrieve the inputs of a job. + """ + job = get_job(request) + results = dict(inputs=[dict(id=get_any_id(_input), value=get_any_value(_input)) for _input in job.inputs]) + return HTTPOk(json=results) + + +@sd.provider_outputs_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ProviderOutputsEndpoint(), response_schemas=sd.get_job_outputs_responses) +@sd.process_outputs_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ProcessOutputsEndpoint(), response_schemas=sd.get_job_outputs_responses) +@sd.job_outputs_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, + schema=sd.JobOutputsEndpoint(), response_schemas=sd.get_job_outputs_responses) +@log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobResultsResponse.description) +def get_job_outputs(request): + # type: (Request) -> HTTPException + """ + Retrieve the outputs of a job. + """ + job = get_job(request) + results = dict(outputs=[dict(id=get_any_id(result), value=get_any_value(result)) for result in job.results]) + return HTTPOk(json=results) + + def get_results(job, container): # type: (Job, AnySettingsContainer) -> JSON """ @@ -269,26 +320,34 @@ def get_results(job, container): return {"outputs": outputs} -@sd.results_full_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.FullResultsEndpoint(), response_schemas=sd.get_job_results_responses) -@sd.results_short_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.ShortResultsEndpoint(), response_schemas=sd.get_job_results_responses) +@sd.provider_results_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.FullResultsEndpoint(), response_schemas=sd.get_job_results_responses) @sd.process_results_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, schema=sd.ProcessResultsEndpoint(), response_schemas=sd.get_job_results_responses) +@sd.job_results_service.get(tags=[sd.TAG_JOBS, sd.TAG_RESULTS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ShortResultsEndpoint(), response_schemas=sd.get_job_results_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobResultsResponse.description) def get_job_results(request): + # type: (Request) -> HTTPException """ Retrieve the results of a job. """ job = get_job(request) + job_status = status.map_status(job.status) + if job_status in status.JOB_STATUS_CATEGORIES[status.STATUS_CATEGORY_RUNNING]: + raise HTTPNotFound(json={ + "code": "ResultsNotReady", + "description": "Job status is '{}'. Results are not yet available.".format(job_status) + }) results = get_results(job, request) return HTTPOk(json=results) -@sd.exceptions_full_service.get(tags=[sd.TAG_JOBS, sd.TAG_EXCEPTIONS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.FullExceptionsEndpoint(), response_schemas=sd.get_exceptions_responses) -@sd.exceptions_short_service.get(tags=[sd.TAG_JOBS, sd.TAG_EXCEPTIONS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.ShortExceptionsEndpoint(), response_schemas=sd.get_exceptions_responses) +@sd.provider_exceptions_service.get(tags=[sd.TAG_JOBS, sd.TAG_EXCEPTIONS, sd.TAG_PROVIDERS], + renderer=OUTPUT_FORMAT_JSON, + schema=sd.FullExceptionsEndpoint(), response_schemas=sd.get_exceptions_responses) +@sd.job_exceptions_service.get(tags=[sd.TAG_JOBS, sd.TAG_EXCEPTIONS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ShortExceptionsEndpoint(), response_schemas=sd.get_exceptions_responses) @sd.process_exceptions_service.get(tags=[sd.TAG_JOBS, sd.TAG_EXCEPTIONS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, schema=sd.ProcessExceptionsEndpoint(), response_schemas=sd.get_exceptions_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobExceptionsResponse.description) @@ -300,10 +359,10 @@ def get_job_exceptions(request): return HTTPOk(json=job.exceptions) -@sd.logs_full_service.get(tags=[sd.TAG_JOBS, sd.TAG_LOGS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.FullLogsEndpoint(), response_schemas=sd.get_logs_responses) -@sd.logs_short_service.get(tags=[sd.TAG_JOBS, sd.TAG_LOGS], renderer=OUTPUT_FORMAT_JSON, - schema=sd.ShortLogsEndpoint(), response_schemas=sd.get_logs_responses) +@sd.provider_logs_service.get(tags=[sd.TAG_JOBS, sd.TAG_LOGS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.FullLogsEndpoint(), response_schemas=sd.get_logs_responses) +@sd.job_logs_service.get(tags=[sd.TAG_JOBS, sd.TAG_LOGS], renderer=OUTPUT_FORMAT_JSON, + schema=sd.ShortLogsEndpoint(), response_schemas=sd.get_logs_responses) @sd.process_logs_service.get(tags=[sd.TAG_JOBS, sd.TAG_LOGS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, schema=sd.ProcessLogsEndpoint(), response_schemas=sd.get_logs_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobLogsResponse.description) @@ -313,11 +372,3 @@ def get_job_logs(request): """ job = get_job(request) return HTTPOk(json=job.logs) - - -# TODO: https://github.com/crim-ca/weaver/issues/18 -# @sd.process_logs_service.get(tags=[sd.TAG_JOBS, sd.TAG_PROCESSES], renderer=OUTPUT_FORMAT_JSON, -# schema=sd.ProcessOutputEndpoint(), response_schemas=sd.get_job_output_responses) -@log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorGetJobOutputResponse.description) -def get_job_output(request): - pass diff --git a/weaver/wps_restapi/processes/__init__.py b/weaver/wps_restapi/processes/__init__.py index 6103055a6..f11ffa2bd 100644 --- a/weaver/wps_restapi/processes/__init__.py +++ b/weaver/wps_restapi/processes/__init__.py @@ -38,7 +38,7 @@ def includeme(config): request_method="GET", renderer=OUTPUT_FORMAT_JSON) config.add_view(p.describe_provider_process, route_name=sd.provider_process_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) - config.add_view(p.submit_provider_job, route_name=sd.jobs_full_service.name, + config.add_view(p.submit_provider_job, route_name=sd.provider_jobs_service.name, request_method="POST", renderer=OUTPUT_FORMAT_JSON) config.add_view(p.get_process_visibility, route_name=sd.process_visibility_service.name, request_method="GET", renderer=OUTPUT_FORMAT_JSON) diff --git a/weaver/wps_restapi/processes/processes.py b/weaver/wps_restapi/processes/processes.py index 108e205b6..83acfc8e0 100644 --- a/weaver/wps_restapi/processes/processes.py +++ b/weaver/wps_restapi/processes/processes.py @@ -45,7 +45,15 @@ from weaver.processes.utils import convert_process_wps_to_db, deploy_process_from_payload, jsonify_output from weaver.status import STATUS_ACCEPTED, STATUS_FAILED, STATUS_STARTED, STATUS_SUCCEEDED, map_status from weaver.store.base import StoreJobs, StoreProcesses, StoreServices -from weaver.utils import get_any_id, get_any_value, get_cookie_headers, get_settings, raise_on_xml_exception, wait_secs +from weaver.utils import ( + get_any_id, + get_any_value, + get_cookie_headers, + get_settings, + raise_on_xml_exception, + transform_json, + wait_secs +) from weaver.visibility import VISIBILITY_PUBLIC, VISIBILITY_VALUES from weaver.wps import get_wps_output_dir, get_wps_output_path, get_wps_output_url, load_pywps_cfg from weaver.wps_restapi import swagger_definitions as sd @@ -341,6 +349,9 @@ def submit_job_handler(request, service_url, is_workflow=False, visibility=None) .format(CONTENT_TYPE_APP_JSON)) try: json_body = request.json_body + if "inputs" in json_body and isinstance(json_body["inputs"], list): + for i in json_body["inputs"]: + transform_json(i, rename={"data": "value"}) # backward compatibility except Exception as ex: raise HTTPBadRequest("Invalid JSON body cannot be decoded for job submission. [{}]".format(ex)) try: @@ -387,9 +398,9 @@ def submit_job_handler(request, service_url, is_workflow=False, visibility=None) return HTTPCreated(location=location, json=body_data) -@sd.jobs_full_service.post(tags=[sd.TAG_PROVIDER_PROCESS, sd.TAG_PROVIDERS, sd.TAG_EXECUTE, sd.TAG_JOBS], - renderer=OUTPUT_FORMAT_JSON, schema=sd.PostProviderProcessJobRequest(), - response_schemas=sd.post_provider_process_job_responses) +@sd.provider_jobs_service.post(tags=[sd.TAG_PROVIDER_PROCESS, sd.TAG_PROVIDERS, sd.TAG_EXECUTE, sd.TAG_JOBS], + renderer=OUTPUT_FORMAT_JSON, schema=sd.PostProviderProcessJobRequest(), + response_schemas=sd.post_provider_process_job_responses) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorPostProviderProcessJobResponse.description) def submit_provider_job(request): """ @@ -683,8 +694,14 @@ def submit_local_job(request): visibility=process.visibility) return resp except InvalidIdentifierValue as ex: - raise HTTPBadRequest(str(ex)) + raise HTTPBadRequest(json={ + "code": InvalidIdentifierValue.__name__, + "description": str(ex) + }) except ProcessNotAccessible: - raise HTTPUnauthorized("Process with id '{!s}' is not accessible.".format(process_id)) + raise HTTPUnauthorized(json={ + "code": "", + "description": "Process with id '{!s}' is not accessible.".format(process_id) + }) except ProcessNotFound: raise HTTPNotFound("The process with id '{!s}' does not exist.".format(process_id)) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index e9030292a..a40194c3e 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -81,57 +81,6 @@ class ExplicitMappingSchema(MapSchema): } URL = "url" -######################################################################### -# API endpoints -######################################################################### - -api_frontpage_uri = "/" -api_swagger_ui_uri = "/api" -api_swagger_json_uri = "/json" -api_versions_uri = "/versions" -api_conformance_uri = "/conformance" - -processes_uri = "/processes" -process_uri = "/processes/{process_id}" -process_package_uri = "/processes/{process_id}/package" -process_payload_uri = "/processes/{process_id}/payload" -process_visibility_uri = "/processes/{process_id}/visibility" -process_jobs_uri = "/processes/{process_id}/jobs" -process_job_uri = "/processes/{process_id}/jobs/{job_id}" -process_quotes_uri = "/processes/{process_id}/quotations" -process_quote_uri = "/processes/{process_id}/quotations/{quote_id}" -process_results_uri = "/processes/{process_id}/jobs/{job_id}/result" -process_exceptions_uri = "/processes/{process_id}/jobs/{job_id}/exceptions" -process_logs_uri = "/processes/{process_id}/jobs/{job_id}/logs" - -providers_uri = "/providers" -provider_uri = "/providers/{provider_id}" - -provider_processes_uri = "/providers/{provider_id}/processes" -provider_process_uri = "/providers/{provider_id}/processes/{process_id}" - -jobs_short_uri = "/jobs" -jobs_full_uri = "/providers/{provider_id}/processes/{process_id}/jobs" -job_full_uri = "/providers/{provider_id}/processes/{process_id}/jobs/{job_id}" -job_exceptions_uri = "/providers/{provider_id}/processes/{process_id}/jobs/{job_id}/exceptions" -job_short_uri = "/jobs/{job_id}" - -quotes_uri = "/quotations" -quote_uri = "/quotations/{quote_id}" -bills_uri = "/bills" -bill_uri = "/bill/{bill_id}" - -results_full_uri = "/providers/{provider_id}/processes/{process_id}/jobs/{job_id}/result" -results_short_uri = "/jobs/{job_id}/result" -result_full_uri = "/providers/{provider_id}/processes/{process_id}/jobs/{job_id}/result/{result_id}" -result_short_uri = "/jobs/{job_id}/result/{result_id}" - -exceptions_full_uri = "/providers/{provider_id}/processes/{process_id}/jobs/{job_id}/exceptions" -exceptions_short_uri = "/jobs/{job_id}/exceptions" - -logs_full_uri = "/providers/{provider_id}/processes/{process_id}/jobs/{job_id}/logs" -logs_short_uri = "/jobs/{job_id}/logs" - ######################################################### # API tags ######################################################### @@ -153,53 +102,57 @@ class ExplicitMappingSchema(MapSchema): TAG_EXCEPTIONS = "Exceptions" TAG_LOGS = "Logs" -############################################################################### +######################################################################### +# API endpoints # These "services" are wrappers that allow Cornice to generate the JSON API ############################################################################### -api_frontpage_service = Service(name="api_frontpage", path=api_frontpage_uri) -api_swagger_ui_service = Service(name="api_swagger_ui", path=api_swagger_ui_uri) -api_swagger_json_service = Service(name="api_swagger_json", path=api_swagger_json_uri) -api_versions_service = Service(name="api_versions", path=api_versions_uri) -api_conformance_service = Service(name="api_conformance", path=api_conformance_uri) - -processes_service = Service(name="processes", path=processes_uri) -process_service = Service(name="process", path=process_uri) -process_package_service = Service(name="process_package", path=process_package_uri) -process_payload_service = Service(name="process_payload", path=process_payload_uri) -process_visibility_service = Service(name="process_visibility", path=process_visibility_uri) -process_jobs_service = Service(name="process_jobs", path=process_jobs_uri) -process_job_service = Service(name="process_job", path=process_job_uri) -process_quotes_service = Service(name="process_quotes", path=process_quotes_uri) -process_quote_service = Service(name="process_quote", path=process_quote_uri) -process_results_service = Service(name="process_results", path=process_results_uri) -process_exceptions_service = Service(name="process_exceptions", path=process_exceptions_uri) -process_logs_service = Service(name="process_logs", path=process_logs_uri) - -providers_service = Service(name="providers", path=providers_uri) -provider_service = Service(name="provider", path=provider_uri) - -provider_processes_service = Service(name="provider_processes", path=provider_processes_uri) -provider_process_service = Service(name="provider_process", path=provider_process_uri) - -jobs_short_service = Service(name="jobs_short", path=jobs_short_uri) -jobs_full_service = Service(name="jobs_full", path=jobs_full_uri) -job_full_service = Service(name="job_full", path=job_full_uri) -job_short_service = Service(name="job_short", path=job_short_uri) - -quotes_service = Service(name="quotes", path=quotes_uri) -quote_service = Service(name="quote", path=quote_uri) -bills_service = Service(name="bills", path=bills_uri) -bill_service = Service(name="bill", path=bill_uri) - -results_full_service = Service(name="results_full", path=results_full_uri) -results_short_service = Service(name="results_short", path=results_short_uri) - -exceptions_full_service = Service(name="exceptions_full", path=exceptions_full_uri) -exceptions_short_service = Service(name="exceptions_short", path=exceptions_short_uri) - -logs_full_service = Service(name="logs_full", path=logs_full_uri) -logs_short_service = Service(name="logs_short", path=logs_short_uri) +api_frontpage_service = Service(name="api_frontpage", path="/") +api_swagger_ui_service = Service(name="api_swagger_ui", path="/api") +api_swagger_json_service = Service(name="api_swagger_json", path="/json") +api_versions_service = Service(name="api_versions", path="/versions") +api_conformance_service = Service(name="api_conformance", path="/conformance") + +quotes_service = Service(name="quotes", path="/quotations") +quote_service = Service(name="quote", path=quotes_service.path + "/{quote_id}") +bills_service = Service(name="bills", path="/bills") +bill_service = Service(name="bill", path=bills_service.path + "/{bill_id}") + +jobs_service = Service(name="jobs", path="/jobs") +job_service = Service(name="job", path=jobs_service.path + "/{job_id}") +job_results_service = Service(name="job_results", path=job_service.path + "/results") +job_exceptions_service = Service(name="job_exceptions", path=job_service.path + "/exceptions") +job_outputs_service = Service(name="job_outputs", path=job_service.path + "/outputs") +job_inputs_service = Service(name="job_inputs", path=job_service.path + "/inputs") +job_logs_service = Service(name="job_logs", path=job_service.path + "/logs") + +processes_service = Service(name="processes", path="/processes") +process_service = Service(name="process", path=processes_service.path + "/{process_id}") +process_quotes_service = Service(name="process_quotes", path=process_service.path + quotes_service.path) +process_quote_service = Service(name="process_quote", path=process_service.path + quote_service.path) +process_visibility_service = Service(name="process_visibility", path=process_service.path + "/visibility") +process_package_service = Service(name="process_package", path=process_service.path + "/package") +process_payload_service = Service(name="process_payload", path=process_service.path + "/payload") +process_jobs_service = Service(name="process_jobs", path=process_service.path + jobs_service.path) +process_job_service = Service(name="process_job", path=process_service.path + job_service.path) +process_results_service = Service(name="process_results", path=process_service.path + job_results_service.path) +process_inputs_service = Service(name="process_inputs", path=process_service.path + job_inputs_service.path) +process_outputs_service = Service(name="process_outputs", path=process_service.path + job_outputs_service.path) +process_exceptions_service = Service(name="process_exceptions", path=process_service.path + job_exceptions_service.path) +process_logs_service = Service(name="process_logs", path=process_service.path + job_logs_service.path) + +providers_service = Service(name="providers", path="/providers") +provider_service = Service(name="provider", path=providers_service.path + "/{provider_id}") +provider_processes_service = Service(name="provider_processes", path=provider_service.path + processes_service.path) +provider_process_service = Service(name="provider_process", path=provider_service.path + process_service.path) +provider_jobs_service = Service(name="provider_jobs", path=provider_service.path + process_jobs_service.path) +provider_job_service = Service(name="provider_job", path=provider_service.path + process_job_service.path) +provider_results_service = Service(name="provider_results", path=provider_service.path + process_results_service.path) +provider_inputs_service = Service(name="provider_inputs", path=provider_service.path + process_inputs_service.path) +provider_outputs_service = Service(name="provider_outputs", path=provider_service.path + process_outputs_service.path) +provider_logs_service = Service(name="provider_logs", path=provider_service.path + process_logs_service.path) +provider_exceptions_service = Service(name="provider_exceptions", + path=provider_service.path + process_exceptions_service.path) ######################################################### # Path parameter definitions @@ -651,6 +604,30 @@ class ShortJobEndpoint(JobPath): header = AcceptHeader() +class ProcessInputsEndpoint(ProcessPath, JobPath): + header = AcceptHeader() + + +class ProviderInputsEndpoint(ProviderPath, ProcessPath, JobPath): + header = AcceptHeader() + + +class JobInputsEndpoint(JobPath): + header = AcceptHeader() + + +class ProcessOutputsEndpoint(ProcessPath, JobPath): + header = AcceptHeader() + + +class ProviderOutputsEndpoint(ProviderPath, ProcessPath, JobPath): + header = AcceptHeader() + + +class JobOutputsEndpoint(JobPath): + header = AcceptHeader() + + class ProcessResultsEndpoint(ProcessPath, JobPath): header = AcceptHeader() @@ -782,15 +759,12 @@ class JobStatusInfo(MappingSchema): jobID = SchemaNode(String(), example="a9d14bf4-84e0-449a-bac8-16e598efe807", description="ID of the job.") status = JobStatusEnum() message = SchemaNode(String(), missing=drop) - # fixme: use href links (https://github.com/crim-ca/weaver/issues/58) [logs/result/exceptions] - logs = SchemaNode(String(), missing=drop) - result = SchemaNode(String(), missing=drop) - exceptions = SchemaNode(String(), missing=drop) expirationDate = SchemaNode(DateTime(), missing=drop) estimatedCompletion = SchemaNode(DateTime(), missing=drop) duration = SchemaNode(String(), missing=drop, description="Duration of the process execution.") nextPoll = SchemaNode(DateTime(), missing=drop) percentCompleted = SchemaNode(Integer(), example=0, validator=Range(min=0, max=100)) + links = JsonLinkList(missing=drop) class JobEntrySchema(OneOfMappingSchema): @@ -890,32 +864,35 @@ class DataEncodingAttributes(MappingSchema): encoding = SchemaNode(String(), missing=drop) -class DataFloat(DataEncodingAttributes): - data = SchemaNode(Float()) +class ValueFloat(DataEncodingAttributes): + value = SchemaNode(Float()) -class DataInteger(DataEncodingAttributes): - data = SchemaNode(Integer()) +class ValueInteger(DataEncodingAttributes): + value = SchemaNode(Integer()) -class DataString(DataEncodingAttributes): - data = SchemaNode(String()) +class ValueString(DataEncodingAttributes): + value = SchemaNode(String()) -class DataBoolean(DataEncodingAttributes): - data = SchemaNode(Boolean()) +class ValueBoolean(DataEncodingAttributes): + value = SchemaNode(Boolean()) class ValueType(OneOfMappingSchema): - _one_of = (DataFloat, - DataInteger, - DataString, - DataBoolean, + _one_of = (ValueFloat, + ValueInteger, + ValueString, + ValueBoolean, Reference) class Input(InputDataType, ValueType): - pass + """ + Default value to be looked for uses key 'value' to conform to OGC API standard. + We still look for 'href', 'data' and 'reference' to remain back-compatible. + """ class InputList(SequenceSchema): @@ -1056,10 +1033,10 @@ class JobOutputsSchema(SequenceSchema): class OutputInfo(OutputDataType, OneOfMappingSchema): - _one_of = (DataFloat, - DataInteger, - DataString, - DataBoolean, + _one_of = (ValueFloat, + ValueInteger, + ValueString, + ValueBoolean, Reference) @@ -1554,23 +1531,37 @@ class InternalServerErrorGetJobStatusResponse(MappingSchema): description = "Unhandled error occurred during provider process description." -class Result(MappingSchema): +class Inputs(MappingSchema): + inputs = InputList() + links = JsonLinkList(missing=drop) + + +class OkGetJobInputsResponse(MappingSchema): + header = JsonHeader() + body = Inputs() + + +class Outputs(MappingSchema): outputs = OutputInfoList() links = JsonLinkList(missing=drop) -class OkGetJobResultsResponse(MappingSchema): +class OkGetJobOutputsResponse(MappingSchema): header = JsonHeader() - body = Result() + body = Outputs() -class InternalServerErrorGetJobResultsResponse(MappingSchema): - description = "Unhandled error occurred during job results listing." +class ResultInfoList(OutputInfoList): + pass -class OkGetOutputResponse(MappingSchema): +class OkGetJobResultsResponse(MappingSchema): header = JsonHeader() - body = JobOutputSchema() + body = ResultInfoList() + + +class InternalServerErrorGetJobResultsResponse(MappingSchema): + description = "Unhandled error occurred during job results listing." class InternalServerErrorGetJobOutputResponse(MappingSchema): @@ -1740,7 +1731,7 @@ class InternalServerErrorGetJobLogsResponse(MappingSchema): } post_provider_responses = { "201": CreatedPostProvider(description="success"), - "400": MappingSchema(description=OWSMissingParameterValue.explanation), + "400": MappingSchema(description=OWSMissingParameterValue.description), "401": UnauthorizedJsonResponseSchema(description="unauthorized"), "500": InternalServerErrorPostProviderResponse(), "501": NotImplementedPostProviderResponse(), @@ -1770,15 +1761,20 @@ class InternalServerErrorGetJobLogsResponse(MappingSchema): "401": UnauthorizedJsonResponseSchema(description="unauthorized"), "500": InternalServerErrorDeleteJobResponse(), } -get_job_results_responses = { - "200": OkGetJobResultsResponse(description="success"), +get_job_inputs_responses = { + "200": OkGetJobInputsResponse(description="success"), "401": UnauthorizedJsonResponseSchema(description="unauthorized"), "500": InternalServerErrorGetJobResultsResponse(), } -get_job_output_responses = { - "200": OkGetOutputResponse(description="success"), +get_job_outputs_responses = { + "200": OkGetJobOutputsResponse(description="success"), "401": UnauthorizedJsonResponseSchema(description="unauthorized"), - "500": InternalServerErrorGetJobOutputResponse(), + "500": InternalServerErrorGetJobResultsResponse(), +} +get_job_results_responses = { + "200": OkGetJobResultsResponse(description="success"), + "401": UnauthorizedJsonResponseSchema(description="unauthorized"), + "500": InternalServerErrorGetJobResultsResponse(), } get_exceptions_responses = { "200": OkGetJobExceptionsResponse(description="success"),