Verify all file object records into a manifest csv
- Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html
index 614870e0a..23649076e 100644
--- a/docs/_build/html/tools/metadata.html
+++ b/docs/_build/html/tools/metadata.html
@@ -6,7 +6,7 @@
Metadata Tools — Gen3 SDK documentation
-
+
@@ -101,7 +101,7 @@ Metadata Tools[source]
Ingest all metadata records into a manifest csv
- Parameters:
diff --git a/docs/_build/html/wss.html b/docs/_build/html/wss.html
index dd97fffe2..275542eba 100644
--- a/docs/_build/html/wss.html
+++ b/docs/_build/html/wss.html
@@ -6,7 +6,7 @@
Gen3 Workspace Storage — Gen3 SDK documentation
-
+
diff --git a/docs/howto/discoveryMetadataTools.md b/docs/howto/discoveryMetadataTools.md
index aaefe0c3a..795af82aa 100644
--- a/docs/howto/discoveryMetadataTools.md
+++ b/docs/howto/discoveryMetadataTools.md
@@ -149,7 +149,7 @@ def test_manual_single_doi(publish_dois=False):
titles = [DigitalObjectIdentifierTitle("Some Example Study in Gen3").as_dict()]
publisher = "Example Gen3 Sponsor"
publication_year = 2023
- doi_type = "Dataset"
+ doi_type_general = "Dataset"
version = 1
doi_metadata = {
@@ -158,7 +158,7 @@ def test_manual_single_doi(publish_dois=False):
"titles": titles,
"publisher": publisher,
"publication_year": publication_year,
- "doi_type": doi_type,
+ "doi_type_general": doi_type_general,
"version": version,
}
diff --git a/gen3/doi.py b/gen3/doi.py
index e5091cf75..582a3a3eb 100644
--- a/gen3/doi.py
+++ b/gen3/doi.py
@@ -56,7 +56,8 @@ def main():
titles=["test"],
publisher="test",
publication_year=2023,
- doi_type="Dataset",
+ doi_type_general="Other",
+ doi_type="Study Record",
url="https://example.com",
version="0.1",
descriptions=[{"description": "this is a test resource"}],
@@ -276,7 +277,7 @@ def persist_doi_metadata_in_gen3(
f"({doi.publication_year}). "
f"{'& '.join([titles.get('title') for titles in doi.titles])} "
f"{str(doi.optional_fields.get('version', '1'))}. "
- f"{doi.publisher}. {doi.doi_type}. {doi.identifier}"
+ f"{doi.publisher}. {doi.doi_type_general}. {doi.identifier}"
)
if additional_metadata.get("citation"):
del additional_metadata["citation"]
@@ -292,7 +293,7 @@ def persist_doi_metadata_in_gen3(
)
metadata[
prefix + "version_information"
- ] = f"This is version {doi.optional_fields.get('version', '1')} of this {doi.doi_type}."
+ ] = f"This is version {doi.optional_fields.get('version', '1')} of this {doi.doi_type_general}."
if additional_metadata.get("version_information"):
del additional_metadata["version_information"]
@@ -460,6 +461,7 @@ def __init__(
titles=None,
publisher=None,
publication_year=None,
+ doi_type_general=None,
doi_type=None,
url=None,
root_url=None,
@@ -500,6 +502,7 @@ def __init__(
Contributor/contributorType/ hostingInstitution for the code repository.
publication_year (int, optional): the year the data or resource was or will be made
publicly available
+ doi_type_general (str, optional): a description of the general type of a resource. (controlled list values, see https://support.datacite.org/docs/datacite-metadata-schema-v44-mandatory-properties#10a-resourcetypegeneral)
doi_type (str, optional): a description of the resource (free-format text)
url (str, optional): the web address of the landing page for the resource
root_url (str, optional): the root url for the landing page, must be used
@@ -517,7 +520,10 @@ def __init__(
self.titles = titles or []
self.publisher = publisher
self.publication_year = publication_year
- self.doi_type = doi_type or DigitalObjectIdentifier.DEFAULT_DOI_TYPE
+ self.doi_type_general = (
+ doi_type_general or DigitalObjectIdentifier.DEFAULT_DOI_TYPE
+ )
+ self.doi_type = doi_type
self.url = url
self.root_url = root_url
self.event = event
@@ -573,10 +579,13 @@ def __init__(
self.url = self._get_url_from_root()
- if doi_type and doi_type not in DigitalObjectIdentifier.RESOURCE_GENERAL_TYPES:
- logging.error(f"{doi_type} is not an accepted resourceTypeGeneral")
+ if (
+ doi_type_general
+ and doi_type_general not in DigitalObjectIdentifier.RESOURCE_GENERAL_TYPES
+ ):
+ logging.error(f"{doi_type_general} is not an accepted resourceTypeGeneral")
raise DataCiteDOIValidationError(
- f"{doi_type} is not an accepted resourceTypeGeneral"
+ f"{doi_type_general} is not an accepted resourceTypeGeneral"
)
def _get_url_from_root(self):
@@ -606,6 +615,8 @@ def as_gen3_metadata(self, prefix=""):
data[prefix + "publisher"] = self.publisher
if self.publication_year:
data[prefix + "publication_year"] = self.publication_year
+ if self.doi_type_general:
+ data[prefix + "resource_type_general"] = self.doi_type_general
if self.doi_type:
data[prefix + "resource_type"] = self.doi_type
if self.url:
@@ -688,8 +699,14 @@ def as_dict(self):
data["data"]["attributes"]["publisher"] = self.publisher
if self.publication_year:
data["data"]["attributes"]["publicationYear"] = self.publication_year
+ if self.doi_type_general:
+ data["data"]["attributes"]["types"] = {
+ "resourceTypeGeneral": self.doi_type_general
+ }
+ else:
+ data["data"]["attributes"]["types"] = {}
if self.doi_type:
- data["data"]["attributes"]["types"] = {"resourceTypeGeneral": self.doi_type}
+ data["data"]["attributes"]["types"].update({"resourceType": self.doi_type})
if self.url:
data["data"]["attributes"]["url"] = self.url
if self.event:
@@ -979,7 +996,7 @@ def __init__(self, description, description_type=TYPE_ABSTRACT):
f"Provided description_type '{description_type}' is NOT in {allowed_types}. "
f"Defaulting to '{DigitalObjectIdentifierDescription.TYPE_ABSTRACT}'."
)
- description_type = DigitalObjectIdentifierDescription.NAME_TYPE_PERSON
+ description_type = DigitalObjectIdentifierDescription.TYPE_ABSTRACT
self.description_type = description_type
diff --git a/gen3/external/nih/dbgap_doi.py b/gen3/external/nih/dbgap_doi.py
index 03ede5fbb..ea8366b86 100644
--- a/gen3/external/nih/dbgap_doi.py
+++ b/gen3/external/nih/dbgap_doi.py
@@ -120,7 +120,7 @@ def get_metadata_for_ids(self, ids):
doi_metadata["publication_year"] = dbgapDOI._get_doi_publication_year(
phsid, dbgap_fhir_metadata
)
- doi_metadata["doi_type"] = "Dataset"
+ doi_metadata["doi_type_general"] = "Dataset"
# publisher is provided
doi_metadata["publisher"] = self.publisher
diff --git a/pyproject.toml b/pyproject.toml
index 59f860954..2fa1a02ad 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,7 @@
[tool.poetry]
name = "gen3"
homepage = "https://gen3.org/"
-version = "4.20.3"
+version = "4.20.4"
description = "Gen3 CLI and Python SDK"
authors = ["Center for Translational Data Science at the University of Chicago "]
license = "Apache-2.0"
diff --git a/tests/test_doi.py b/tests/test_doi.py
index 341a9d8d9..e5f0dd87b 100644
--- a/tests/test_doi.py
+++ b/tests/test_doi.py
@@ -21,12 +21,13 @@
@pytest.mark.parametrize(
- "expect_failure,doi_type,prefix,identifier,creators,titles,publisher,publication_year,descriptions",
+ "expect_failure,doi_type_general,doi_type,prefix,identifier,creators,titles,publisher,publication_year,descriptions",
[
# everything normal
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -39,6 +40,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -51,6 +53,20 @@
(
False,
"Dataset",
+ None,
+ "10.12345",
+ "ID1234",
+ ["test"],
+ ["test title"],
+ "test",
+ 2000,
+ ["this is a test description"],
+ ),
+ # providing both DOI Type General and DOI Type
+ (
+ False,
+ "Other",
+ "Study Record",
"10.12345",
"ID1234",
["test"],
@@ -63,6 +79,7 @@
(
True,
"Not a valid doi type",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -77,6 +94,7 @@
"Dataset",
None,
None,
+ None,
["test"],
["test title"],
"test",
@@ -87,6 +105,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test", "!@#$% ^&*|\\O({@[< >?;:]})", "Name. MIDDLE Last"],
@@ -99,6 +118,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -111,6 +131,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -123,6 +144,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -135,6 +157,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -147,6 +170,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test", "test2", "test3"],
@@ -159,6 +183,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -171,6 +196,7 @@
(
False,
"Dataset",
+ None,
"10.12345",
"ID1234",
["test"],
@@ -187,6 +213,7 @@ def test_create_doi(
expect_failure,
prefix,
identifier,
+ doi_type_general,
doi_type,
creators,
titles,
@@ -211,6 +238,7 @@ def test_create_doi(
titles=titles,
publisher=publisher,
publication_year=publication_year,
+ doi_type_general=doi_type_general,
doi_type=doi_type,
url="https://example.com",
version="0.1",
@@ -225,6 +253,7 @@ def test_create_doi(
titles=titles,
publisher=publisher,
publication_year=publication_year,
+ doi_type_general=doi_type_general,
doi_type=doi_type,
url="https://example.com",
version="0.1",
@@ -263,7 +292,11 @@ def _mock_request(url, **kwargs):
assert title in data_in_request.get("titles")
assert data_in_request.get("publisher") == publisher
assert data_in_request.get("publicationYear") == publication_year
- assert data_in_request.get("types", {}).get("resourceTypeGeneral") == doi_type
+ assert (
+ data_in_request.get("types", {}).get("resourceTypeGeneral")
+ == doi_type_general
+ )
+ assert data_in_request.get("types", {}).get("resourceType") == doi_type
for description in descriptions:
assert description in data_in_request.get("descriptions")
@@ -329,7 +362,8 @@ def test_doi_metadata_persist(
titles = [DigitalObjectIdentifierTitle("Some Example Study in Gen3").as_dict()]
publisher = "Example Gen3 Sponsor"
publication_year = 2023
- doi_type = "Dataset"
+ doi_type_general = "Dataset"
+ doi_type = ""
version = 1
doi_metadata = {
@@ -338,6 +372,7 @@ def test_doi_metadata_persist(
"titles": titles,
"publisher": publisher,
"publication_year": publication_year,
+ "doi_type_general": doi_type_general,
"doi_type": doi_type,
"version": version,
}
@@ -424,7 +459,7 @@ def _raise():
"doi_titles": "Some Example Study in Gen3",
"doi_publisher": "Example Gen3 Sponsor",
"doi_publication_year": 2023,
- "doi_resource_type": "Dataset",
+ "doi_resource_type_general": "Dataset",
"doi_url": f"foobar/{identifier}/",
"doi_version": 1,
"doi_is_available": "Yes",