diff --git a/docker-compose.yml b/docker-compose.yml index 5c0febb..8005877 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: context: . container_name: nldi-py ports: - - "8080:80" + - "8081:80" environment: NLDI_PATH: /api/nldi NLDI_URL: http://localhost:8081/api/nldi diff --git a/nldi/api.py b/nldi/api.py index 4fc6e3b..72aa407 100644 --- a/nldi/api.py +++ b/nldi/api.py @@ -310,7 +310,8 @@ def get_comid_by_id(self, request: Union[APIRequest, Any], headers = request.get_response_headers(**HEADERS) try: - content = self.flowline_lookup.get(identifier) + feature = self.flowline_lookup.get(identifier) + features = [feature, ] except ProviderConnectionError: msg = 'connection error (check logs)' return self.get_exception( @@ -327,7 +328,8 @@ def get_comid_by_id(self, request: Union[APIRequest, Any], HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + content = stream_j2_template('FeatureCollection.j2', features) + return headers, HTTPStatus.OK, content @pre_process def get_hydrolocation(self, request: Union[APIRequest, Any] @@ -416,7 +418,8 @@ def get_comid_by_position(self, request: Union[APIRequest, Any] 'NoApplicableCode', msg) try: - content = self.flowline_lookup.get(identifier) + feature = self.flowline_lookup.get(identifier) + features = [feature, ] except ProviderQueryError: msg = 'query error (check logs)' return self.get_exception( @@ -428,7 +431,8 @@ def get_comid_by_position(self, request: Union[APIRequest, Any] HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + content = stream_j2_template('FeatureCollection.j2', features) + return headers, HTTPStatus.OK, content @pre_process def get_source_features(self, request: Union[APIRequest, Any], @@ -468,21 +472,39 @@ def get_source_features(self, request: Union[APIRequest, Any], 'NoApplicableCode', msg) plugin = self.load_plugin('FeatureLookup', source=source) - try: - content = plugin.get(identifier) if identifier else plugin.query() - except ProviderQueryError: - msg = 'query error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) - except ProviderItemNotFoundError: - msg = f'The feature source \'{source_name}\' has not been crawled.' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) + if identifier: + try: + feature = plugin.get(identifier) + features = [feature, ] + except ProviderQueryError: + msg = 'query error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderItemNotFoundError: + msg = f'The source \'{source_name}\' has no item {identifier}.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + else: + try: + features = plugin.query() + except ProviderQueryError: + msg = 'query error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderItemNotFoundError: + msg = f'The source \'{source_name}\' has not been crawled.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + # content = stream_j2_template('FeatureGraph.j2', features) + content = stream_j2_template('FeatureCollection.j2', features) - _ = stream_j2_template('FeatureCollection.j2', content) - return headers, HTTPStatus.OK, _ + return headers, HTTPStatus.OK, content @pre_process def get_basin(self, request: Union[APIRequest, Any], @@ -530,11 +552,11 @@ def get_basin(self, request: Union[APIRequest, Any], HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) - _ = request.params.get('simplified', 'True') - simplified = _.lower() == 'true' + _ = request.params.get('simplified', 'True').lower() == 'true' + simplified = _ - _ = request.params.get('splitCatchment', 'False') - splitCatchment = _.lower() == 'true' # noqa + _ = request.params.get('splitCatchment', 'False').lower() == 'true' + splitCatchment = _ if isPoint and splitCatchment: LOGGER.debug('Split Catchment') @@ -592,8 +614,6 @@ def get_navigation_info(self, request: Union[APIRequest, Any], 'downstreamMain': url_join(nav_url, 'DM'), 'downstreamDiversions': url_join(nav_url, 'DD'), } - if source_name == 'comid': - content.update({'pointToPoint': url_join(nav_url, 'PP')}) return headers, HTTPStatus.OK, to_json(content, self.pretty_print) @@ -622,10 +642,9 @@ def get_navigation_info(self, request: Union[APIRequest, Any], return headers, HTTPStatus.OK, to_json(content, self.pretty_print) @pre_process - def get_navigation(self, request: Union[APIRequest, Any], - source_name: str, identifier: str, - nav_mode: str, data_source: str - ) -> Tuple[dict, int, str]: + def get_fl_navigation(self, request: Union[APIRequest, Any], + source_name: str, identifier: str, nav_mode: str + ) -> Tuple[dict, int, str]: """ Provide navigation query @@ -633,7 +652,6 @@ def get_navigation(self, request: Union[APIRequest, Any], :param source_name: NLDI source name :param identifier: NLDI Source feature identifier :param nav_mode: NLDI Navigation mode - :param data_source: NLDI output source_name :returns: tuple of headers, status code, content """ @@ -642,21 +660,12 @@ def get_navigation(self, request: Union[APIRequest, Any], headers = request.get_response_headers(**HEADERS) - try: - distance = request.params['distance'] - except KeyError: - msg = 'Required request parameter \'distance\' is not present.' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) - start_comid = None source_name = source_name.lower() if source_name == 'comid': try: self.flowline_lookup.get(identifier) - start_comid = int(identifier) except ProviderQueryError: msg = 'query error (check logs)' return self.get_exception( @@ -668,11 +677,20 @@ def get_navigation(self, request: Union[APIRequest, Any], HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) + start_comid = int(identifier) + else: try: source = self.crawler_source.get(source_name) - plugin = self.load_plugin('FeatureLookup', source=source) - feature = next(plugin.get(identifier)) + except ProviderItemNotFoundError: + msg = f'The feature source \'{source_name}\' does not exist.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + plugin = self.load_plugin('FeatureLookup', source=source) + try: + feature = plugin.get(identifier) start_comid = int(feature['properties']['comid']) except ProviderQueryError: msg = 'query error (check logs)' @@ -685,46 +703,132 @@ def get_navigation(self, request: Union[APIRequest, Any], HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) except ProviderItemNotFoundError: - msg = f'The feature source \'{source_name}\' does not exist.' + msg = f'The source \'{source_name}\' has no item {identifier}.' return self.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) - nav_results = self.func.get_navigation( - nav_mode, start_comid, distance) + try: + distance = float(request.params['distance']) + except KeyError: + msg = 'Required request parameter \'distance\' is not present.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ValueError: + msg = 'Required request parameter \'distance\' must be a number.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + nav_results = self.func.get_navigation(nav_mode, start_comid, distance) + features = self.flowline_lookup.lookup_navigation(nav_results) + content = stream_j2_template('FeatureCollection.j2', features) - source2_name = data_source.lower() - if source2_name == 'flowlines': + return headers, HTTPStatus.OK, content + + @pre_process + def get_navigation(self, request: Union[APIRequest, Any], + source_name: str, identifier: str, + nav_mode: str, data_source: str + ) -> Tuple[dict, int, str]: + """ + Provide navigation query + + :param request: A request object + :param source_name: NLDI source name + :param identifier: NLDI Source feature identifier + :param nav_mode: NLDI Navigation mode + :param data_source: NLDI output source_name + + :returns: tuple of headers, status code, content + """ + if not request.is_valid(): + return self.get_exception(request) + + headers = request.get_response_headers(**HEADERS) + + start_comid = None + source_name = source_name.lower() + + if source_name == 'comid': try: - content = self.flowline_lookup.lookup_navigation(nav_results) + self.flowline_lookup.get(identifier) except ProviderQueryError: msg = 'query error (check logs)' return self.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) except ProviderItemNotFoundError: - msg = f'The feature source \'{source2_name}\' does not exist.' + msg = f'The comid source \'{identifier}\' does not exist.' return self.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) + + start_comid = int(identifier) + else: try: - source2 = self.crawler_source.get(source2_name) - plugin = self.load_plugin('FeatureLookup', source=source2) - content = plugin.lookup_navigation(nav_results) + source = self.crawler_source.get(source_name) + except ProviderItemNotFoundError: + msg = f'The feature source \'{source_name}\' does not exist.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + plugin = self.load_plugin('FeatureLookup', source=source) + try: + feature = plugin.get(identifier) + start_comid = int(feature['properties']['comid']) except ProviderQueryError: msg = 'query error (check logs)' return self.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) + except (KeyError, IndexError): + msg = f'The feature {identifier} from source \'{source_name}\' is not indexed.' # noqa + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) except ProviderItemNotFoundError: - msg = f'The feature source \'{source2_name}\' does not exist.' + msg = f'The source \'{source_name}\' has no item {identifier}.' return self.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) - _ = stream_j2_template('FeatureCollection.j2', content) - return headers, HTTPStatus.OK, _ + source2_name = data_source.lower() + try: + source2 = self.crawler_source.get(source2_name) + plugin = self.load_plugin('FeatureLookup', source=source2) + except ProviderQueryError: + msg = 'query error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderItemNotFoundError: + msg = f'The feature source \'{source2_name}\' does not exist.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + try: + distance = float(request.params['distance']) + except KeyError: + msg = 'Required request parameter \'distance\' is not present.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ValueError: + msg = 'Required request parameter \'distance\' must be a number.' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + nav_results = self.func.get_navigation(nav_mode, start_comid, distance) + features = plugin.lookup_navigation(nav_results) + content = stream_j2_template('FeatureCollection.j2', features) + + return headers, HTTPStatus.OK, content def get_exception(self, status, headers, format_, code, description) -> Tuple[dict, int, str]: diff --git a/nldi/flask_app.py b/nldi/flask_app.py index d5292c6..8f261a2 100644 --- a/nldi/flask_app.py +++ b/nldi/flask_app.py @@ -193,6 +193,21 @@ def get_navigation_info(source_name=None, identifier=None, nav_mode=None): API_.get_navigation_info(request, source_name, identifier, nav_mode)) +@BLUEPRINT.route('/linked-data///navigation//flowlines') # noqa +def get_flowline_navigation(source_name=None, identifier=None, nav_mode=None): # noqa + """ + Data source flowline navigation endpoint + + :param source_name: NLDI input source name + :param identifier: NLDI Source feature identifier + :param nav_mode: NLDI Navigation mode + + :returns: HTTP response + """ + return get_response(API_.get_fl_navigation( + request, source_name, identifier, nav_mode)) + + @BLUEPRINT.route('/linked-data///navigation//') # noqa def get_navigation(source_name=None, identifier=None, nav_mode=None, data_source=None): # noqa """ diff --git a/nldi/functions/__init__.py b/nldi/functions/__init__.py index 18080cc..248ff11 100644 --- a/nldi/functions/__init__.py +++ b/nldi/functions/__init__.py @@ -37,8 +37,8 @@ from nldi.functions.basin import get_basin from nldi.functions.navigate import get_navigation +from nldi.lookup import _ENGINE_STORE -_ENGINE_STORE = {} LOGGER = logging.getLogger(__name__) diff --git a/nldi/lookup/__init__.py b/nldi/lookup/__init__.py index e99bc52..82e8384 100644 --- a/nldi/lookup/__init__.py +++ b/nldi/lookup/__init__.py @@ -28,3 +28,5 @@ # ================================================================= """Module containing the models for the NLDI data lookup""" + +_ENGINE_STORE = {} diff --git a/nldi/lookup/base.py b/nldi/lookup/base.py index 30fb567..951682e 100644 --- a/nldi/lookup/base.py +++ b/nldi/lookup/base.py @@ -29,12 +29,13 @@ from contextlib import contextmanager import logging -from sqlalchemy import create_engine +from sqlalchemy import create_engine, func from sqlalchemy.engine import URL from sqlalchemy.orm import sessionmaker from typing import Iterable -_ENGINE_STORE = {} +from nldi.lookup import _ENGINE_STORE + LOGGER = logging.getLogger(__name__) @@ -57,6 +58,7 @@ def __init__(self, provider_def): LOGGER.debug('Initialising BaseLookup') # Read table information from database + self.geom_field = None self.id_field = None self.table_model = None self.db_search_path = [] @@ -97,7 +99,11 @@ def session(self): try: Session = sessionmaker(bind=self._engine) session = Session() - yield session.query(self.table_model) + if self.geom_field: + geom = func.ST_AsGeoJSON(self.geom_field).label('geom') + yield session.query(self.table_model, geom) + else: + yield session.query(self.table_model) finally: session.close() diff --git a/nldi/lookup/feature.py b/nldi/lookup/feature.py index bd94de4..3ff014d 100644 --- a/nldi/lookup/feature.py +++ b/nldi/lookup/feature.py @@ -27,9 +27,8 @@ # # ================================================================= -from geoalchemy2.shape import to_shape +import json import logging -import shapely from typing import Iterable @@ -61,7 +60,8 @@ def __init__(self, provider_def): url_join(self.base_url, 'linked-data', self.source_name) super().__init__(provider_def) - self.id_field = 'identifier' + self.geom_field = FeatureSourceModel.location + self.id_field = FeatureSourceModel.identifier self.table_model = FeatureSourceModel self.table_model.__tablename__ = f'feature_{self.source_name}' @@ -72,19 +72,20 @@ def get(self, identifier: str): with self.session() as session: # Retrieve data from database as feature item = (session - .get(identifier)) + .filter(self.id_field == identifier) + .first()) if item is None: msg = f'No such item: {self.id_field}={identifier}' raise ProviderItemNotFoundError(msg) - yield self._sqlalchemy_to_feature(item) + return self._sqlalchemy_to_feature(item) def query(self): crawler_source_id = self.source.get('crawler_source_id') crawler_source_id_ = FeatureSourceModel.crawler_source_id - LOGGER.debug(f'Feching features for: {crawler_source_id}') + LOGGER.debug(f'Fetching features for source id: {crawler_source_id}') with self.session() as session: # Retrieve data from database as feature query = (session @@ -118,36 +119,35 @@ def lookup_navigation(self, comids: Iterable[str]): yield self._sqlalchemy_to_feature(item) def _sqlalchemy_to_feature(self, item): - - if item.location: - shapely_geom = to_shape(item.location) - geojson_geom = shapely.geometry.mapping(shapely_geom) - geometry = geojson_geom + if self.geom_field: + (feature, geom) = item + geometry = json.loads(geom) else: + feature = item geometry = None try: - mainstem = item.mainstem_lookup.uri + mainstem = feature.mainstem_lookup.uri except AttributeError: mainstem = '' navigation = \ - url_join(self.relative_url, item.identifier, 'navigation') + url_join(self.relative_url, feature.identifier, 'navigation') return { 'type': 'Feature', 'properties': { - 'identifier': item.identifier, - 'name': item.name, - 'source': item.crawler_source.source_suffix, - 'sourceName': item.crawler_source.source_name, - 'comid': item.comid, - 'type': item.crawler_source.feature_type, - 'uri': item.uri, - 'reachcode': item.reachcode, - 'measure': item.measure, + 'identifier': feature.identifier, + 'name': feature.name, + 'source': feature.crawler_source.source_suffix, + 'sourceName': feature.crawler_source.source_name, + 'comid': feature.comid, + 'type': feature.crawler_source.feature_type, + 'uri': feature.uri, + 'reachcode': feature.reachcode, + 'measure': feature.measure, 'navigation': navigation, 'mainstem': mainstem }, - 'geometry': geometry, + 'geometry': geometry } diff --git a/nldi/lookup/flowline.py b/nldi/lookup/flowline.py index f316017..e3adfb0 100644 --- a/nldi/lookup/flowline.py +++ b/nldi/lookup/flowline.py @@ -27,9 +27,8 @@ # # ================================================================= -from geoalchemy2.shape import to_shape +import json import logging -import shapely from typing import Iterable from nldi.lookup.base import BaseLookup, ProviderItemNotFoundError @@ -57,21 +56,24 @@ def __init__(self, provider_def): self.relative_url = url_join(self.base_url, 'linked-data/comid') super().__init__(provider_def) - self.id_field = 'featureid' + self.geom_field = FlowlineModel.shape + self.id_field = FlowlineModel.nhdplus_comid self.table_model = FlowlineModel def get(self, identifier: str): LOGGER.debug(f'Fetching comid with id: {identifier}') with self.session() as session: # Retrieve data from database as feature - item = session.get(identifier) + item = (session + .filter(self.id_field == identifier) + .first()) if item is None: msg = f'No comid found for: {identifier}.' raise ProviderItemNotFoundError(msg) - LOGGER.debug(f'Intersection with {item.nhdplus_comid}') - yield self._sqlalchemy_to_feature(item) + LOGGER.debug(f'Intersection with {item[0].nhdplus_comid}') + return self._sqlalchemy_to_feature(item) def lookup_navigation(self, comids: Iterable[str]): with self.session() as session: @@ -89,11 +91,11 @@ def lookup_navigation(self, comids: Iterable[str]): yield self._sqlalchemy_to_feature(item) def _sqlalchemy_to_feature(self, item): - if item.shape: - shapely_geom = to_shape(item.shape) - geojson_geom = shapely.geometry.mapping(shapely_geom) - geometry = geojson_geom + if self.geom_field: + (feature, geom) = item + geometry = json.loads(geom) else: + feature = item geometry = None try: @@ -101,16 +103,16 @@ def _sqlalchemy_to_feature(self, item): except AttributeError: mainstem = '' - navigation = url_join(self.relative_url, - item.nhdplus_comid, 'navigation') + navigation = url_join( + self.relative_url, feature.nhdplus_comid, 'navigation') return { 'type': 'Feature', 'properties': { - 'identifier': item.permanent_identifier, + 'identifier': feature.permanent_identifier, 'source': 'comid', 'sourceName': 'NHDPlus comid', - 'comid': item.nhdplus_comid, + 'comid': feature.nhdplus_comid, 'mainstem': mainstem, 'navigation': navigation }, diff --git a/nldi/lookup/mainstem.py b/nldi/lookup/mainstem.py index 8498b9a..4a7068e 100644 --- a/nldi/lookup/mainstem.py +++ b/nldi/lookup/mainstem.py @@ -50,14 +50,17 @@ def __init__(self, provider_def): """ LOGGER.debug('Initialising Mainstem Lookup.') super().__init__(provider_def) - self.id_field = 'nhdpv2_comid' + self.id_field = MainstemLookupModel.nhdpv2_comid self.table_model = MainstemLookupModel def get(self, identifier: str): LOGGER.debug(f'Fetching mainstem for: {identifier}') with self.session() as session: # Retrieve data from database as feature - item = session.get(identifier) + item = (session + .filter(self.id_field == identifier) + .first()) + if item is None: msg = f'No mainstem found: {self.id_field}={identifier}.' raise ProviderItemNotFoundError(msg) diff --git a/nldi/openapi/__init__.py b/nldi/openapi/__init__.py index a3b0e99..4aab97f 100644 --- a/nldi/openapi/__init__.py +++ b/nldi/openapi/__init__.py @@ -462,7 +462,7 @@ def get_oas(cfg): 'operationId': f'{src_title}NavigationDataSource', 'parameters': [ *parameters, - {'$ref': '#/components/parameters/navigationModePP'}, + {'$ref': '#/components/parameters/navigationMode'}, { 'name': 'dataSource', 'in': 'path', @@ -473,9 +473,7 @@ def get_oas(cfg): 'enum': _sources } }, - {'$ref': '#/components/parameters/distance'}, - {'$ref': '#/components/parameters/stopComid'}, - {'$ref': '#/components/parameters/legacy'} + {'$ref': '#/components/parameters/distance'} ], 'responses': { '200': { @@ -513,12 +511,10 @@ def get_oas(cfg): 'operationId': f'{src_title}NavigationFlowlines', 'parameters': [ *parameters, - {'$ref': '#/components/parameters/navigationModePP'}, + {'$ref': '#/components/parameters/navigationMode'}, {'$ref': '#/components/parameters/distance'}, - {'$ref': '#/components/parameters/stopComid'}, {'$ref': '#/components/parameters/trimStart'}, - {'$ref': '#/components/parameters/trimTolerance'}, - {'$ref': '#/components/parameters/legacy'} + {'$ref': '#/components/parameters/trimTolerance'} ], 'responses': { '200': { diff --git a/nldi/schemas/nldi_data.py b/nldi/schemas/nldi_data.py index 61dee64..d64b736 100644 --- a/nldi/schemas/nldi_data.py +++ b/nldi/schemas/nldi_data.py @@ -43,7 +43,7 @@ class CrawlerSourceModel(BaseModel): crawler_source_id = Column(Integer, primary_key=True) source_name = Column(String(500)) source_suffix = Column(String(10), - server_default=text("lower(source_suffix)")) + server_default=text('lower(source_suffix)')) source_uri = Column(String(256)) feature_id = Column(String(256)) feature_name = Column(String(256))