update to latest OGC API - Records specification (#988)

* update to latest OGC API - Records specification
* update tests
* fix deprecation warnings, removed unused import
* remove Python 3.8 from GitHub Actions
* update copyright year

Author: tomkralidis

.github/workflows/main.yml

@@ -21,10 +21,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"
-            toxenv: "py38-sqlite"
-          - python-version: "3.9"
-            toxenv: "py39-sqlite"
           - python-version: "3.10"
             toxenv: "py310-sqlite"
           - python-version: "3.11"

.github/workflows/vulnerabilities.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: '3.10'
           architecture: x64
       - name: Checkout pycsw
         uses: actions/checkout@master

pycsw/core/repository.py

@@ -38,10 +38,7 @@
 from time import sleep
 
 from shapely.wkt import loads
-try:
-    from shapely.errors import ReadingError
-except Exception:
-    from shapely.geos import ReadingError
+from shapely.errors import ShapelyError
 
 from sqlalchemy import create_engine, func, __version__, select
 from sqlalchemy.exc import OperationalError
@@ -306,7 +303,8 @@ def describe(self):
 
         properties = {
             'geometry': {
-                '$ref': 'https://geojson.org/schema/Polygon.json'
+                '$ref': 'https://geojson.org/schema/Polygon.json',
+                'x-ogc-role': 'primary-geometry'
             }
         }
 
@@ -318,6 +316,9 @@ def describe(self):
                 'title': i.name
             }
 
+            if i.name == 'identifier':
+                properties[i.name]['x-ogc-role'] = 'id'
+
             try:
                 properties[i.name]['type'] = type_mappings[str(i.type)]
             except Exception as err:
@@ -636,7 +637,7 @@ def query_spatial(bbox_data_wkt, bbox_input_wkt, predicate, distance):
         else:
             raise RuntimeError(
                 'Invalid spatial query predicate: %s' % predicate)
-    except (AttributeError, ValueError, ReadingError, TypeError):
+    except (AttributeError, ValueError, ShapelyError, TypeError):
         result = False
     return "true" if result else "false"
 

pycsw/ogc/api/records.py

@@ -45,7 +45,7 @@
 from pycsw.core.pygeofilter_evaluate import to_filter
 from pycsw.core.util import bind_url, get_today_and_now, jsonify_links, load_custom_repo_mappings, str2bool, wkt2geom
 from pycsw.ogc.api.oapi import gen_oapi
-from pycsw.ogc.api.util import match_env_var, render_j2_template, to_json
+from pycsw.ogc.api.util import match_env_var, render_j2_template, to_json, to_rfc3339
 
 LOGGER = logging.getLogger(__name__)
 
@@ -109,7 +109,7 @@ def __init__(self, config: dict):
         try:
             self.limit = int(self.config['server']['maxrecords'])
         except KeyError:
-            self.limit= 10
+            self.limit = 10
         LOGGER.debug(f'limit: {self.limit}')
 
         repo_filter = self.config['repository'].get('filter')
@@ -466,7 +466,7 @@ def queryables(self, headers_, args, collection='metadata:main'):
             headers_['Content-Type'] = 'application/schema+json'
 
         if collection not in self.get_all_collections():
-            msg = f'Invalid collection'
+            msg = 'Invalid collection'
             LOGGER.exception(msg)
             return self.get_exception(400, headers_, 'InvalidParameterValue', msg)
 
@@ -506,6 +506,11 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
         :returns: tuple of headers, status code, content
         """
 
+        LOGGER.debug(f'Request args: {args.keys()}')
+        LOGGER.debug('converting request argument names to lower case')
+        args = {k.lower(): v for k, v in args.items()}
+        LOGGER.debug(f'Request args (lower case): {args.keys()}')
+
         headers_['Content-Type'] = self.get_content_type(headers_, args)
 
         reserved_query_params = [
@@ -539,7 +544,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
         collections = []
 
         if collection not in self.get_all_collections():
-            msg = f'Invalid collection'
+            msg = 'Invalid collection'
             LOGGER.exception(msg)
             return self.get_exception(400, headers_, 'InvalidParameterValue', msg)
 
@@ -830,7 +835,7 @@ def item(self, headers_, args, collection, item):
         headers_['Content-Type'] = self.get_content_type(headers_, args)
 
         if collection not in self.get_all_collections():
-            msg = f'Invalid collection'
+            msg = 'Invalid collection'
             LOGGER.exception(msg)
             return self.get_exception(400, headers_, 'InvalidParameterValue', msg)
 
@@ -992,6 +997,7 @@ def get_collection_info(self, collection_name: str = 'metadata:main',
 
         collection_info = {
             'id': id_,
+            'type': 'catalog',
             'title': title,
             'description': description,
             'itemType': 'record',
@@ -1127,22 +1133,29 @@ def record2json(record, url, collection, mode='ogcapi-records'):
         'id': record.identifier,
         'type': 'Feature',
         'geometry': None,
-        'time': record.date,
         'properties': {},
         'links': []
     }
 
+    try:
+        dt, dt_type = to_rfc3339(record.date)
+        record_dict['time'] = {
+            dt_type: dt
+        }
+    except Exception:
+        record_dict['time'] = None
+
     # todo; for keywords with a scheme use the theme property
-    themes = []
     if record.topicategory:
+        themes = []
         themes.append({'concepts': [record.topicategory],
                        'scheme': 'https://standards.iso.org/iso/19139/resources/gmxCodelists.xml#MD_TopicCategoryCode'})
-    record_dict['properties']['themes'] = themes
+        record_dict['properties']['themes'] = themes
 
     if record.otherconstraints:
-        if isinstance(record.otherconstraints, str):
+        if isinstance(record.otherconstraints, str) and record.otherconstraints not in [None, 'None']:
             record.otherconstraints = [record.otherconstraints]
-        record_dict['properties']['license'] = ", ".join(record.otherconstraints)
+            record_dict['properties']['license'] = ", ".join(record.otherconstraints)
 
     record_dict['properties']['updated'] = record.insert_date
 
@@ -1168,7 +1181,7 @@ def record2json(record, url, collection, mode='ogcapi-records'):
         record_dict['properties']['description'] = record.abstract
 
     if record.format:
-        record_dict['properties']['formats'] = [record.format]
+        record_dict['properties']['formats'] = [{'name': record.format}]
 
     if record.keywords:
         record_dict['properties']['keywords'] = [x for x in record.keywords.split(',')]
@@ -1181,58 +1194,67 @@ def record2json(record, url, collection, mode='ogcapi-records'):
                     rcnt.append({
                         'name': cnt['name'],
                         'organization': cnt.get('organization', ''),
-                        'positionName': cnt.get('position', ''),
-                        'roles': [
-                            {'name': cnt.get('role', '')}
-                        ],
-                        'contactInfo': {
-                            'phone': {'work': cnt.get('phone', '')},
-                            'email': {'work': cnt.get('email', '')},
-                            'address': {
-                                'work': {
-                                    'deliveryPoint': cnt.get('address', ''),
-                                    'city': cnt.get('city', ''),
-                                    'administrativeArea': cnt.get('region', ''),
-                                    'postalCode': cnt.get('postcode', ''),
-                                    'country': cnt.get('country', ''),
-                                }
-                            },
-                            'url': cnt.get('onlineresource', '')
-                        }
+                        'position': cnt.get('position', ''),
+                        'roles': [cnt.get('role', '')],
+                        'phones': [{
+                            'value': cnt.get('phone', '')
+                        }],
+                        'emails': [{
+                            'value': cnt.get('email', '')
+                        }],
+                        'addresses': [{
+                            'deliveryPoint': [cnt.get('address', '')],
+                            'city': cnt.get('city', ''),
+                            'administrativeArea': cnt.get('region', ''),
+                            'postalCode': cnt.get('postcode', ''),
+                            'country': cnt.get('country', '')
+                        }],
+                        'links': [{
+                            'href': cnt.get('onlineresource')
+                        }]
                     })
                 except Exception as err:
                     LOGGER.exception(f"failed to parse contact of {record.identifier}: {err}")
         except Exception as err:
             LOGGER.exception(f"failed to parse contacts json of {record.identifier}: {err}")
-        record_dict['properties']['providers'] = rcnt
+
+        record_dict['properties']['contacts'] = rcnt
 
     if record.themes not in [None, '', 'null']:
-        ogcapiThemes = []
+        ogcapi_themes = []
         # For a scheme, prefer uri over label
         # OWSlib currently uses .keywords_object for keywords with url, see https://github.com/geopython/OWSLib/pull/765
         try:
             for theme in json.loads(record.themes):
                 try:
-                    ogcapiThemes.append({
+                    ogcapi_themes.append({
                         'scheme': theme['thesaurus'].get('url', theme['thesaurus'].get('title', '')),
                         'concepts': [c for c in theme.get('keywords_object', []) if c not in [None, '']]
                     })
                 except Exception as err:
                     LOGGER.exception(f"failed to parse theme of {record.identifier}: {err}")
         except Exception as err:
             LOGGER.exception(f"failed to parse themes json of {record.identifier}: {err}")
-        record_dict['properties']['themes'] = ogcapiThemes
+
+        record_dict['properties']['themes'] = ogcapi_themes
 
     if record.links:
         rdl = record_dict['links']
 
         for link in jsonify_links(record.links):
+            if link['url'] in [None, 'None']:
+                LOGGER.debug(f'Skipping null link: {link}')
+                continue
+
             link2 = {
-                'href': link['url'],
-                'name': link.get('name'),
-                'description': link.get('description'),
-                'type': link.get('protocol')
+                'href': link['url']
             }
+            if link.get('name') not in [None, 'None']:
+                link2['name'] = link['name']
+            if link.get('description') not in [None, 'None']:
+                link2['description'] = link['description']
+            if link.get('protocol') not in [None, 'None']:
+                link2['procotol'] = link['protocol']
             if 'rel' in link:
                 link2['rel'] = link['rel']
             elif link['protocol'] == 'WWW:LINK-1.0-http--image-thumbnail':
@@ -1289,18 +1311,28 @@ def record2json(record, url, collection, mode='ogcapi-records'):
     if record.time_begin or record.time_end:
         if record.time_end not in [None, '']:
             if record.time_begin not in [None, '']:
-                record_dict['time'] = [record.time_begin, record.time_end]
+                begin, _ = to_rfc3339(record.time_begin)
+                end, _ = to_rfc3339(record.time_end)
+                record_dict['time'] = {
+                    'interval': [begin, end]
+                }
             else:
-                record_dict['time'] = record.time_end
+                end, end_type = to_rfc3339(record.time_end)
+                record_dict['time'] = {
+                    end_type: end
+                }
         else:
-            record_dict['time'] = record.time_begin
+            begin, begin_type = to_rfc3339(record.time_begin)
+            record_dict['time'] = {
+                begin_type: begin
+            }
 
     if mode == 'stac-api':
-        record_dict['properties']['datetime'] = record.date
+        record_dict['properties']['datetime'] = to_rfc3339(record.date)
 
         if None not in [record.time_begin, record.time_end]:
-            record_dict['properties']['start_datetime'] = record.time_begin
-            record_dict['properties']['end_datetime'] = record.time_end
+            record_dict['properties']['start_datetime'] = to_rfc3339(record.time_begin)
+            record_dict['properties']['end_datetime'] = to_rfc3339(record.time_end)
 
     return record_dict
 
@@ -1322,7 +1354,7 @@ def build_anytext(name, value):
     tokens = value.split(',')
 
     if len(tokens) == 1 and ' ' not in value:  # single term
-        LOGGER.debug(f'Single term with no spaces')
+        LOGGER.debug('Single term with no spaces')
         return f"{name} ILIKE '%{value}%'"
 
     for token in tokens:

pycsw/ogc/api/util.py

@@ -39,7 +39,9 @@
 import os
 import pathlib
 import re
+from typing import Union
 
+from dateutil.parser import parse as dparse
 from jinja2 import Environment, FileSystemLoader
 from jinja2.exceptions import TemplateNotFound
 import yaml
@@ -82,12 +84,14 @@ def json_serial(obj):
     """
     helper function to convert to JSON non-default
     types (source: https://stackoverflow.com/a/22238613)
+
     :param obj: `object` to be evaluated
+
     :returns: JSON non-default type to `str`
     """
 
     if isinstance(obj, (datetime, date, time)):
-        return obj.isoformat()
+        return obj.isoformat() + 'Z'
     elif isinstance(obj, bytes):
         try:
             LOGGER.debug('Returning as UTF-8 decoded bytes')
@@ -218,3 +222,28 @@ def render_j2_template(config, template, data):
             raise
 
     return template.render(config=config, data=data, version=__version__)
+
+
+def to_rfc3339(value: str) -> Union[tuple, None]:
+    """
+    Helper function to convert a date/datetime into
+    RFC3339
+
+    :param value: `str` of date/datetime value
+
+    :returns: `tuple` of `datetime` of RFC3339 value and date type
+    """
+
+    try:
+        dt = dparse(value)  # TODO TIMEZONE)
+    except Exception as err:
+        msg = f'Parse error: {err}'
+        LOGGER.error(msg)
+        return 'date', None
+
+    if len(value) < 11:
+        dt_type = 'date'
+    else:
+        dt_type = 'date-time'
+
+    return dt, dt_type

pycsw/ogc/gml/gml3.py

@@ -3,7 +3,7 @@
 #
 # Authors: Tom Kralidis <[email protected]>
 #
-# Copyright (c) 2015 Tom Kralidis
+# Copyright (c) 2024 Tom Kralidis
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -214,11 +214,11 @@ def transform(self, src, dest):
 
         geom = loads(self.wkt)
 
-        if geom.type == 'Point':
+        if geom.geom_type == 'Point':
             newgeom = Point(transformer.transform(geom.x, geom.y))
             wkt2 = newgeom.wkt
 
-        elif geom.type == 'LineString':
+        elif geom.geom_type == 'LineString':
             for vertice in list(geom.coords):
                 newgeom = transformer.transform(vertice[0], vertice[1])
                 vertices.append(newgeom)
@@ -227,7 +227,7 @@ def transform(self, src, dest):
 
             wkt2 = linestring.wkt
 
-        elif geom.type == 'Polygon':
+        elif geom.geom_type == 'Polygon':
             for vertice in list(geom.exterior.coords):
                 newgeom = transformer.transform(vertice[0], vertice[1])
                 vertices.append(newgeom)