fix(harvest): use dct.created for creation date and store dct.issued

udata/core/dataservices/models.py

@@ -90,6 +90,10 @@ class HarvestMetadata(db.EmbeddedDocument):
     created_at = field(
         db.DateTimeField(), description="Date of the creation as provided by the harvested catalog"
     )
+    issued_at = field(
+        db.DateTimeField(),
+        description="Date of formal issuance (publication) as provided by the harvested catalog",
+    )
     last_update = field(db.DateTimeField(), description="Date of the last harvesting")
     archived_at = field(db.DateTimeField())
 

udata/core/dataservices/rdf.py

@@ -65,7 +65,13 @@ def dataservice_from_rdf(
 
     dataservice.harvest.uri = d.identifier.toPython() if isinstance(d.identifier, URIRef) else None
     dataservice.harvest.remote_url = remote_url_from_rdf(d)
-    dataservice.harvest.created_at = rdf_value(d, DCT.issued)
+    created_at = rdf_value(d, DCT.created)
+    issued_at = rdf_value(d, DCT.issued)
+    # fallback on issuance date if no creation date found
+    if not created_at and issued_at:
+        created_at = issued_at
+    dataservice.harvest.created_at = created_at
+    dataservice.harvest.issued_at = issued_at
     dataservice.metadata_modified_at = rdf_value(d, DCT.modified)
 
     dataservice.tags = themes_from_rdf(d)

udata/core/dataset/api_fields.py

@@ -45,6 +45,11 @@
         "created_at": fields.ISODateTime(
             description="The dataset harvested creation date", allow_null=True, readonly=True
         ),
+        "issued_at": fields.ISODateTime(
+            description="The dataset harvested issuance (publication) date",
+            allow_null=True,
+            readonly=True,
+        ),
         "modified_at": fields.ISODateTime(
             description="The dataset harvest last modification date", allow_null=True, readonly=True
         ),
@@ -72,6 +77,11 @@
         "created_at": fields.ISODateTime(
             description="The resource harvested creation date", allow_null=True, readonly=True
         ),
+        "issued_at": fields.ISODateTime(
+            description="The dataset harvested issuance (publication) date",
+            allow_null=True,
+            readonly=True,
+        ),
         "modified_at": fields.ISODateTime(
             description="The resource harvest last modification date",
             allow_null=True,

udata/core/dataset/models.py

@@ -71,6 +71,7 @@ def get_json_ld_extra(key, value):
 class HarvestDatasetMetadata(DynamicEmbeddedDocument):
     backend = db.StringField()
     created_at = db.DateTimeField()
+    issued_at = db.DateTimeField()
     modified_at = db.DateTimeField()
     source_id = db.StringField()
     remote_id = db.StringField()
@@ -86,6 +87,7 @@ class HarvestDatasetMetadata(DynamicEmbeddedDocument):
 class HarvestResourceMetadata(DynamicEmbeddedDocument):
     created_at = db.DateTimeField()
     modified_at = db.DateTimeField()
+    issued_at = db.DateTimeField()
     uri = db.StringField()
 
 

udata/core/dataset/rdf.py

@@ -590,13 +590,18 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
 
     identifier = rdf_value(distrib, DCT.identifier)
     uri = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
-    created_at = rdf_value(distrib, DCT.issued)
+    created_at = rdf_value(distrib, DCT.created)
+    issued_at = rdf_value(distrib, DCT.issued)
+    # fallback on issuance date if no creation date found
+    if not created_at and issued_at:
+        created_at = issued_at
     modified_at = rdf_value(distrib, DCT.modified)
 
     if not resource.harvest:
         resource.harvest = HarvestResourceMetadata()
     resource.harvest.created_at = created_at
     resource.harvest.modified_at = modified_at
+    resource.harvest.issued_at = issued_at
     resource.harvest.dct_identifier = identifier
     resource.harvest.uri = uri
 
@@ -678,7 +683,11 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     identifier = rdf_value(d, DCT.identifier)
     uri = d.identifier.toPython() if isinstance(d.identifier, URIRef) else None
     remote_url = remote_url_from_rdf(d)
-    created_at = rdf_value(d, DCT.issued)
+    created_at = rdf_value(d, DCT.created)
+    issued_at = rdf_value(d, DCT.issued)
+    # fallback on issuance date if no creation date found
+    if not created_at and issued_at:
+        created_at = issued_at
     modified_at = rdf_value(d, DCT.modified)
 
     if not dataset.harvest:
@@ -687,6 +696,7 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     dataset.harvest.uri = uri
     dataset.harvest.remote_url = remote_url
     dataset.harvest.created_at = created_at
+    dataset.harvest.issued_at = issued_at
     dataset.harvest.modified_at = modified_at
 
     return dataset

udata/harvest/tests/dcat/catalog.xml

@@ -20,7 +20,8 @@
         <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcterms:publisher rdf:resource="http://data.test.org/organizations/1"/>
-        <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
+        <dcterms:created rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:created>
+        <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2017-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:description>Dataset 3 description</dcterms:description>
         <dcat:keyword>Tag 1</dcat:keyword>
         <dcat:theme rdf:resource="http://data.europa.eu/bna/c_dd313021"/>
@@ -68,7 +69,7 @@
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 1</dcat:keyword>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
-        <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
+        <dcterms:created rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:created>
         <dcterms:identifier>1</dcterms:identifier>
         <dcterms:hasPart rdf:resource="http://data.test.org/datasets/1/resources/3"/>
       </dcat:Dataset>
@@ -84,7 +85,7 @@
         <dcterms:temporal rdf:resource="file:///base/data/home/apps/s%7Erdf-translator/2.408516547054015808/temporal-2"/>
         <dcat:contactPoint rdf:resource="http://data.test.org/contacts/1"/>
         <owl:versionInfo>1.0</owl:versionInfo>
-        <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
+        <dcterms:created rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:created>
         <dcat:landingPage>http://data.test.org/datasets/2</dcat:landingPage>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 3</dcat:keyword>

udata/harvest/tests/test_dcat_backend.py

@@ -494,6 +494,7 @@ def test_xml_catalog(self, rmock):
         assert dataset.harvest.remote_url == "http://data.test.org/datasets/3"
         assert dataset.harvest.remote_id == "3"
         assert dataset.harvest.created_at.date() == date(2016, 12, 14)
+        assert dataset.harvest.issued_at.date() == date(2017, 12, 14)
         assert dataset.harvest.modified_at.date() == date(2016, 12, 14)
         assert dataset.frequency == "daily"
         assert dataset.description == "Dataset 3 description"
@@ -827,7 +828,8 @@ def test_geo2france(self, rmock):
                 "usage-des-sols",
             ]
         )
-        assert dataset.harvest.created_at.date() == date(2017, 10, 7)
+        assert dataset.harvest.created_at.date() == date(2013, 3, 8)
+        assert dataset.harvest.issued_at.date() == date(2017, 10, 7)
         assert dataset.spatial.geom == {
             "type": "MultiPolygon",
             "coordinates": [
@@ -921,3 +923,7 @@ def test_geo_ide(self):
         assert dataset.extras["dcat"].get("rights") is None
         for resource in dataset.resources:
             assert resource.extras["dcat"].get("rights") is None
+
+        # dates are correctly mapped
+        assert dataset.harvest.issued_at == date(2017, 10, 7)
+        assert dataset.harvest.created_at == date(2003, 6, 2)