Improve RFC 3339 datetime handling

Dockerfile.docs

@@ -1,19 +1,21 @@
 FROM python:3.8-slim
 
+# build-essential is required to build a wheel for ciso8601
+RUN apt update && apt install -y build-essential
+
 RUN python -m pip install --upgrade pip
 RUN python -m pip install mkdocs mkdocs-material pdocs
 
 COPY . /opt/src
 
 WORKDIR /opt/src
 
-RUN python -m pip install -e \
+RUN python -m pip install \
     stac_fastapi/api \
     stac_fastapi/types \
     stac_fastapi/extensions \
     stac_fastapi/sqlalchemy
 
-
 CMD ["pdocs", \
      "as_markdown", \
      "--output_dir", \

README.md

@@ -62,6 +62,7 @@ pip install -e stac_fastapi/pgstac
 ```
 
 ## Local Development
+
 Use docker-compose to deploy the application, migrate the database, and ingest some example data:
 ```bash
 docker-compose build
@@ -73,11 +74,23 @@ docker-compose up app-sqlalchemy
 docker-compose up app-pgstac
 ```
 
-For local development it is often more convenient to run the application outside of docker-compose:
+For local development it is often more convenient to run the application outside docker-compose:
 ```bash
 make docker-run
 ```
 
+Before commit, install the [pre-commit](https://pre-commit.com) hooks with:
+
+```shell
+pre-commit install
+```
+
+The pre-commit hooks can be run manually with:
+
+```shell
+pre-commit run --all-files
+```
+
 #### Note to Docker for Windows users
 
 You'll need to enable experimental features on Docker for Windows in order to run the docker-compose, due to the "--platform" flag that is required to allow the project to run on some Apple architectures. To do this, open Docker Desktop, go to settings, select "Docker Engine", and modify the configuration JSON to have `"experimental": true`.

stac_fastapi/pgstac/stac_fastapi/pgstac/core.py

@@ -186,7 +186,7 @@ async def item_collection(
         Called with `GET /collections/{collection_id}/items`
 
         Args:
-            id: id of the collection.
+            collection_id: id of the collection.
             limit: number of items to return.
             token: pagination token.
 

stac_fastapi/pgstac/tests/resources/test_item.py

@@ -1,18 +1,19 @@
 import json
 import uuid
-from datetime import datetime, timedelta
+from datetime import timedelta
 from typing import Callable
 from urllib.parse import parse_qs, urljoin, urlparse
 
 import pystac
 import pytest
 from httpx import AsyncClient
+from pystac.utils import datetime_to_str
 from shapely.geometry import Polygon
 from stac_pydantic import Collection, Item
-from stac_pydantic.shared import DATETIME_RFC339
 from starlette.requests import Request
 
 from stac_fastapi.pgstac.models.links import CollectionLinks
+from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime
 
 
 @pytest.mark.asyncio
@@ -402,14 +403,14 @@ async def test_item_search_temporal_query_post(
     )
     assert resp.status_code == 200
 
-    item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
     print(item_date)
     item_date = item_date + timedelta(seconds=1)
 
     params = {
         "collections": [test_item["collection"]],
         "intersects": test_item["geometry"],
-        "datetime": item_date.strftime(DATETIME_RFC339),
+        "datetime": datetime_to_str(item_date),
     }
 
     resp = await app_client.post("/search", json=params)
@@ -437,14 +438,15 @@ async def test_item_search_temporal_window_post(
     )
     assert resp.status_code == 200
 
-    item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
     item_date_before = item_date - timedelta(seconds=1)
     item_date_after = item_date + timedelta(seconds=1)
 
     params = {
         "collections": [test_item["collection"]],
-        "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}",
+        "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}",
     }
+
     resp = await app_client.post("/search", json=params)
     resp_json = resp.json()
     assert len(resp_json["features"]) == 1
@@ -455,34 +457,24 @@ async def test_item_search_temporal_window_post(
 async def test_item_search_temporal_open_window(
     app_client, load_test_data, load_test_collection
 ):
-    """Test POST search with open spatio-temporal query (core)"""
-    test_item = load_test_data("test_item.json")
-    resp = await app_client.post(
-        f"/collections/{test_item['collection']}/items", json=test_item
-    )
-    assert resp.status_code == 200
-
-    # Add second item with a different datetime.
-    second_test_item = load_test_data("test_item2.json")
-    resp = await app_client.post(
-        f"/collections/{test_item['collection']}/items", json=second_test_item
-    )
-    assert resp.status_code == 200
-
     params = {
-        "collections": [test_item["collection"]],
         "datetime": "../..",
     }
     resp = await app_client.post("/search", json=params)
-    resp_json = resp.json()
-    assert len(resp_json["features"]) == 2
+    assert resp.status_code == 400
+
+    params = {
+        "datetime": "/",
+    }
+    resp = await app_client.post("/search", json=params)
+    assert resp.status_code == 400
 
 
 @pytest.mark.asyncio
 async def test_item_search_sort_post(app_client, load_test_data, load_test_collection):
     """Test POST search with sorting (sort extension)"""
     first_item = load_test_data("test_item.json")
-    item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"])
     resp = await app_client.post(
         f"/collections/{first_item['collection']}/items", json=first_item
     )
@@ -491,7 +483,7 @@ async def test_item_search_sort_post(app_client, load_test_data, load_test_colle
     second_item = load_test_data("test_item.json")
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
-    second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339)
+    second_item["properties"]["datetime"] = datetime_to_str(another_item_date)
     resp = await app_client.post(
         f"/collections/{second_item['collection']}/items", json=second_item
     )
@@ -601,13 +593,13 @@ async def test_item_search_temporal_window_get(
     )
     assert resp.status_code == 200
 
-    item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
     item_date_before = item_date - timedelta(seconds=1)
     item_date_after = item_date + timedelta(seconds=1)
 
     params = {
         "collections": test_item["collection"],
-        "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}",
+        "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}",
     }
     resp = await app_client.get("/search", params=params)
     resp_json = resp.json()
@@ -619,7 +611,7 @@ async def test_item_search_temporal_window_get(
 async def test_item_search_sort_get(app_client, load_test_data, load_test_collection):
     """Test GET search with sorting (sort extension)"""
     first_item = load_test_data("test_item.json")
-    item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"])
     resp = await app_client.post(
         f"/collections/{first_item['collection']}/items", json=first_item
     )
@@ -628,7 +620,7 @@ async def test_item_search_sort_get(app_client, load_test_data, load_test_collec
     second_item = load_test_data("test_item.json")
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
-    second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339)
+    second_item["properties"]["datetime"] = datetime_to_str(another_item_date)
     resp = await app_client.post(
         f"/collections/{second_item['collection']}/items", json=second_item
     )

stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py

@@ -349,13 +349,14 @@ def post_search(
                     # Non-interval date ex. "2000-02-02T00:00:00.00Z"
                     if len(dts) == 1:
                         query = query.filter(self.item_table.datetime == dts[0])
-                    elif ".." not in search_request.datetime:
+                    # is there a benefit to between instead of >= and <= ?
+                    elif dts[0] not in ["", ".."] and dts[1] not in ["", ".."]:
                         query = query.filter(self.item_table.datetime.between(*dts))
                     # All items after the start date
-                    elif dts[0] != "..":
+                    elif dts[0] not in ["", ".."]:
                         query = query.filter(self.item_table.datetime >= dts[0])
                     # All items before the end date
-                    elif dts[1] != "..":
+                    elif dts[1] not in ["", ".."]:
                         query = query.filter(self.item_table.datetime <= dts[1])
 
                 # Query fields

stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py

@@ -1,17 +1,17 @@
 """Serializers."""
 import abc
 import json
-from datetime import datetime
 from typing import TypedDict
 
 import attr
 import geoalchemy2 as ga
-from stac_pydantic.shared import DATETIME_RFC339
+from pystac.utils import datetime_to_str
 
 from stac_fastapi.sqlalchemy.models import database
 from stac_fastapi.types import stac as stac_types
 from stac_fastapi.types.config import Settings
 from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links
+from stac_fastapi.types.rfc3339 import now_to_rfc3339_str, rfc3339_str_to_datetime
 
 
 @attr.s  # type:ignore
@@ -55,7 +55,7 @@ def db_to_stac(cls, db_model: database.Item, base_url: str) -> stac_types.Item:
             # Use getattr to accommodate extension namespaces
             field_value = getattr(db_model, field.split(":")[-1])
             if field == "datetime":
-                field_value = field_value.strftime(DATETIME_RFC339)
+                field_value = datetime_to_str(field_value)
             properties[field] = field_value
         item_id = db_model.id
         collection_id = db_model.collection_id
@@ -101,12 +101,12 @@ def stac_to_db(
             # Use getattr to accommodate extension namespaces
             field_value = stac_data["properties"][field]
             if field == "datetime":
-                field_value = datetime.strptime(field_value, DATETIME_RFC339)
+                field_value = rfc3339_str_to_datetime(field_value)
             indexed_fields[field.split(":")[-1]] = field_value
 
             # TODO: Exclude indexed fields from the properties jsonb field to prevent duplication
 
-            now = datetime.utcnow().strftime(DATETIME_RFC339)
+            now = now_to_rfc3339_str()
             if "created" not in stac_data["properties"]:
                 stac_data["properties"]["created"] = now
             stac_data["properties"]["updated"] = now

stac_fastapi/sqlalchemy/tests/resources/test_item.py

@@ -9,11 +9,12 @@
 
 import pystac
 from pydantic.datetime_parse import parse_datetime
+from pystac.utils import datetime_to_str
 from shapely.geometry import Polygon
-from stac_pydantic.shared import DATETIME_RFC339
 
 from stac_fastapi.sqlalchemy.core import CoreCrudClient
 from stac_fastapi.types.core import LandingPageMixin
+from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime
 
 
 def test_create_and_delete_item(app_client, load_test_data):
@@ -419,13 +420,13 @@ def test_item_search_temporal_query_post(app_client, load_test_data):
     )
     assert resp.status_code == 200
 
-    item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
     item_date = item_date + timedelta(seconds=1)
 
     params = {
         "collections": [test_item["collection"]],
         "intersects": test_item["geometry"],
-        "datetime": f"../{item_date.strftime(DATETIME_RFC339)}",
+        "datetime": f"../{datetime_to_str(item_date)}",
     }
     resp = app_client.post("/search", json=params)
     resp_json = resp.json()
@@ -440,14 +441,14 @@ def test_item_search_temporal_window_post(app_client, load_test_data):
     )
     assert resp.status_code == 200
 
-    item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
     item_date_before = item_date - timedelta(seconds=1)
     item_date_after = item_date + timedelta(seconds=1)
 
     params = {
         "collections": [test_item["collection"]],
         "intersects": test_item["geometry"],
-        "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}",
+        "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}",
     }
     resp = app_client.post("/search", json=params)
     resp_json = resp.json()
@@ -462,20 +463,15 @@ def test_item_search_temporal_open_window(app_client, load_test_data):
     )
     assert resp.status_code == 200
 
-    params = {
-        "collections": [test_item["collection"]],
-        "intersects": test_item["geometry"],
-        "datetime": "../..",
-    }
-    resp = app_client.post("/search", json=params)
-    resp_json = resp.json()
-    assert resp_json["features"][0]["id"] == test_item["id"]
+    for dt in ["/", "../", "/..", "../.."]:
+        resp = app_client.post("/search", json={"datetime": dt})
+        assert resp.status_code == 400
 
 
 def test_item_search_sort_post(app_client, load_test_data):
     """Test POST search with sorting (sort extension)"""
     first_item = load_test_data("test_item.json")
-    item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"])
     resp = app_client.post(
         f"/collections/{first_item['collection']}/items", json=first_item
     )
@@ -484,7 +480,7 @@ def test_item_search_sort_post(app_client, load_test_data):
     second_item = load_test_data("test_item.json")
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
-    second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339)
+    second_item["properties"]["datetime"] = datetime_to_str(another_item_date)
     resp = app_client.post(
         f"/collections/{second_item['collection']}/items", json=second_item
     )
@@ -563,14 +559,14 @@ def test_item_search_temporal_window_get(app_client, load_test_data):
     )
     assert resp.status_code == 200
 
-    item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
     item_date_before = item_date - timedelta(seconds=1)
     item_date_after = item_date + timedelta(seconds=1)
 
     params = {
         "collections": test_item["collection"],
         "bbox": ",".join([str(coord) for coord in test_item["bbox"]]),
-        "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}",
+        "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}",
     }
     resp = app_client.get("/search", params=params)
     resp_json = resp.json()
@@ -580,7 +576,7 @@ def test_item_search_temporal_window_get(app_client, load_test_data):
 def test_item_search_sort_get(app_client, load_test_data):
     """Test GET search with sorting (sort extension)"""
     first_item = load_test_data("test_item.json")
-    item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339)
+    item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"])
     resp = app_client.post(
         f"/collections/{first_item['collection']}/items", json=first_item
     )
@@ -589,7 +585,7 @@ def test_item_search_sort_get(app_client, load_test_data):
     second_item = load_test_data("test_item.json")
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
-    second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339)
+    second_item["properties"]["datetime"] = datetime_to_str(another_item_date)
     resp = app_client.post(
         f"/collections/{second_item['collection']}/items", json=second_item
     )

stac_fastapi/types/setup.py

@@ -10,6 +10,8 @@
     "attrs",
     "pydantic[dotenv]",
     "stac_pydantic==2.0.*",
+    "pystac==1.*",
+    "ciso8601~=2.2.0",
 ]
 
 extra_reqs = {