fix annotation publications list, add SIMULATED to microscope manufac…

…turer enum, and add UniProtKB id support to annotation object
chanzuckerberg · Aug 21, 2024 · 57e99e0 · 57e99e0
1 parent f21d4f3
commit 57e99e0
Show file tree

Hide file tree

Showing 12 changed files with 259 additions and 120 deletions.
diff --git a/ingestion_tools/dataset_configs/template.yaml b/ingestion_tools/dataset_configs/template.yaml
@@ -8,7 +8,7 @@ annotations: OPTIONAL
     annotation_method: REQUIRED, STRING
     annotation_object: REQUIRED
       description: OPTIONAL, STRING
-      id: REQUIRED, STRING (GO_ID)
+      id: REQUIRED, STRING (GO_ID / UNIPROT_ID)
       name: REQUIRED, STRING
       state: OPTIONAL, STRING
     annotation_publications: OPTIONAL, STRING (DOI / EMPIAR / EMDB / PDB IDs)

diff --git a/schema/api/v1.0.0/codegen/api_models_materialized.yaml b/schema/api/v1.0.0/codegen/api_models_materialized.yaml
@@ -295,6 +295,12 @@ types:
     from_schema: cdp-api-models
     base: str
     pattern: ^GO:[0-9]{7}$
+  UNIPROT_ID:
+    name: UNIPROT_ID
+    description: A UniProt identifier
+    from_schema: cdp-api-models
+    base: str
+    pattern: ^UniProtKB:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}+$
   WORMBASE_ID:
     name: WORMBASE_ID
     description: A WormBase identifier
@@ -336,19 +342,19 @@ types:
     description: A Protein Data Bank identifier
     from_schema: cdp-api-models
     base: str
-    pattern: ^pdb[0-9a-zA-Z]{4,8}$
+    pattern: ^PDB-[0-9a-zA-Z]{4,8}$
   EMPIAR_EMDB_PDB_LIST:
     name: EMPIAR_EMDB_PDB_LIST
     description: A list of EMPIAR, EMDB, and PDB identifiers
     from_schema: cdp-api-models
     base: str
-    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$
+    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$
   EMPIAR_EMDB_DOI_PDB_LIST:
     name: EMPIAR_EMDB_DOI_PDB_LIST
     description: A list of EMPIAR, EMDB, DOI, and PDB identifiers
     from_schema: cdp-api-models
     base: str
-    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8}))*$
+    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8}))*$
 enums:
   tomogram_type:
     name: tomogram_type
@@ -489,6 +495,9 @@ enums:
       JEOL:
         text: JEOL
         description: JEOL Ltd.
+      SIMULATED:
+        text: SIMULATED
+        description: Simulated data
   fiducial_alignment_status_enum:
     name: fiducial_alignment_status_enum
     description: Fiducial Alignment method
@@ -1118,8 +1127,8 @@ classes:
         inlined_as_list: true
       annotation_publication:
         name: annotation_publication
-        description: List of publication IDs (EMPIAR, EMDB, DOI) that describe this
-          annotation method. Comma separated.
+        description: List of publication IDs (EMPIAR, EMDB, DOI, PDB) that describe
+          this annotation method. Comma separated.
         from_schema: cdp-api-models
         exact_mappings:
         - cdp-common:annotation_publications
@@ -1130,7 +1139,7 @@ classes:
         range: EMPIAR_EMDB_DOI_PDB_LIST
         inlined: true
         inlined_as_list: true
-        pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8}))*$
+        pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8}))*$
       annotation_method:
         name: annotation_method
         description: Describe how the annotation is made (e.g. Manual, crYoLO, Positive
@@ -1173,11 +1182,13 @@ classes:
         owner: Annotation
         domain_of:
         - Annotation
-        range: GO_ID
         required: true
         inlined: true
         inlined_as_list: true
-        pattern: ^GO:[0-9]{7}$
+        pattern: (^GO:[0-9]{7}$)|(^UniProtKB:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}+$)
+        any_of:
+        - range: GO_ID
+        - range: UNIPROT_ID
       object_name:
         name: object_name
         description: Name of the object being annotated (e.g. ribosome, nuclear pore
@@ -2055,7 +2066,7 @@ classes:
         recommended: true
         inlined: true
         inlined_as_list: true
-        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)
+        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)
       related_database_links:
         name: related_database_links
         description: Comma-separated list of related database links for the dataset.
@@ -2487,7 +2498,7 @@ classes:
         required: true
         inlined: true
         inlined_as_list: true
-        pattern: (^FEI$)|(^TFS$)|(^JEOL$)
+        pattern: (^FEI$)|(^TFS$)|(^JEOL$)|(^SIMULATED$)
       microscope_model:
         name: microscope_model
         description: Microscope model name
@@ -3682,7 +3693,7 @@ classes:
         recommended: true
         inlined: true
         inlined_as_list: true
-        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)
+        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)
       related_database_links:
         name: related_database_links
         description: Comma-separated list of related database links for the dataset.

diff --git a/schema/api/v2.0.0/codegen/api_models_materialized.yaml b/schema/api/v2.0.0/codegen/api_models_materialized.yaml
@@ -295,6 +295,12 @@ types:
     from_schema: cdp-api
     base: string
     pattern: ^GO:[0-9]{7}$
+  UNIPROT_ID:
+    name: UNIPROT_ID
+    description: A UniProt identifier
+    from_schema: cdp-api
+    base: str
+    pattern: ^UniProtKB:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}+$
   WORMBASE_ID:
     name: WORMBASE_ID
     description: A WormBase identifier
@@ -336,19 +342,19 @@ types:
     description: A Protein Data Bank identifier
     from_schema: cdp-api
     base: string
-    pattern: ^pdb[0-9a-zA-Z]{4,8}$
+    pattern: ^PDB-[0-9a-zA-Z]{4,8}$
   EMPIAR_EMDB_PDB_LIST:
     name: EMPIAR_EMDB_PDB_LIST
     description: A list of EMPIAR, EMDB, and PDB identifiers
     from_schema: cdp-api
     base: string
-    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$
+    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$
   EMPIAR_EMDB_DOI_PDB_LIST:
     name: EMPIAR_EMDB_DOI_PDB_LIST
     description: A list of EMPIAR, EMDB, DOI, and PDB identifiers
     from_schema: cdp-api
     base: string
-    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8}))*$
+    pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8}))*$
 enums:
   annotation_file_source_enum:
     name: annotation_file_source_enum
@@ -492,6 +498,9 @@ enums:
       JEOL:
         text: JEOL
         description: JEOL Ltd.
+      SIMULATED:
+        text: SIMULATED
+        description: Simulated data
   fiducial_alignment_status_enum:
     name: fiducial_alignment_status_enum
     description: Fiducial Alignment method
@@ -737,7 +746,7 @@ classes:
         owner: AuthorEntityMixin
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
       primary_author_status:
@@ -751,7 +760,7 @@ classes:
         owner: AuthorEntityMixin
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
   APIDateStampedEntityMixin:
@@ -1232,7 +1241,7 @@ classes:
         owner: AnnotationAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
       primary_author_status:
@@ -1246,7 +1255,7 @@ classes:
         owner: AnnotationAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
   AnnotationFile:
@@ -1537,8 +1546,8 @@ classes:
         inlined_as_list: true
       annotation_publication:
         name: annotation_publication
-        description: List of publication IDs (EMPIAR, EMDB, DOI) that describe this
-          annotation method. Comma separated.
+        description: List of publication IDs (EMPIAR, EMDB, DOI, PDB) that describe
+          this annotation method. Comma separated.
         from_schema: cdp-api
         exact_mappings:
         - cdp-common:annotation_publications
@@ -1549,7 +1558,7 @@ classes:
         range: EMPIAR_EMDB_DOI_PDB_LIST
         inlined: true
         inlined_as_list: true
-        pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|pdb[0-9a-zA-Z]{4,8}))*$
+        pattern: ^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|(doi:)?10\.[0-9]{4,9}/[-._;()/:a-zA-Z0-9]+|PDB-[0-9a-zA-Z]{4,8}))*$
       annotation_method:
         name: annotation_method
         description: Describe how the annotation is made (e.g. Manual, crYoLO, Positive
@@ -1592,11 +1601,13 @@ classes:
         owner: Annotation
         domain_of:
         - Annotation
-        range: GO_ID
         required: true
         inlined: true
         inlined_as_list: true
-        pattern: ^GO:[0-9]{7}$
+        pattern: (^GO:[0-9]{7}$)|(^UniProtKB:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}+$)
+        any_of:
+        - range: GO_ID
+        - range: UNIPROT_ID
       object_name:
         name: object_name
         description: Name of the object being annotated (e.g. ribosome, nuclear pore
@@ -1957,7 +1968,7 @@ classes:
         owner: DatasetAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
       primary_author_status:
@@ -1971,7 +1982,7 @@ classes:
         owner: DatasetAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
   DatasetFunding:
@@ -2458,7 +2469,7 @@ classes:
         recommended: true
         inlined: true
         inlined_as_list: true
-        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)
+        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)
       related_database_links:
         name: related_database_links
         description: Comma-separated list of related database links for the dataset.
@@ -2663,7 +2674,7 @@ classes:
         owner: DepositionAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
       primary_author_status:
@@ -2677,7 +2688,7 @@ classes:
         owner: DepositionAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
   Deposition:
@@ -2861,7 +2872,7 @@ classes:
         recommended: true
         inlined: true
         inlined_as_list: true
-        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)
+        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)
       related_database_links:
         name: related_database_links
         description: Comma-separated list of related database links for the dataset.
@@ -3828,7 +3839,7 @@ classes:
         required: true
         inlined: true
         inlined_as_list: true
-        pattern: (^FEI$)|(^TFS$)|(^JEOL$)
+        pattern: (^FEI$)|(^TFS$)|(^JEOL$)|(^SIMULATED$)
       microscope_model:
         name: microscope_model
         description: Microscope model name
@@ -4333,7 +4344,7 @@ classes:
         owner: TomogramAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
       primary_author_status:
@@ -4347,7 +4358,7 @@ classes:
         owner: TomogramAuthor
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
   TomogramVoxelSpacing:
@@ -5036,7 +5047,7 @@ classes:
         domain_of:
         - DateStampedEntityMixin
         - APIDateStampedEntityMixin
-        range: string
+        range: date
         required: true
         inlined: true
         inlined_as_list: true
@@ -5051,7 +5062,7 @@ classes:
         domain_of:
         - DateStampedEntityMixin
         - APIDateStampedEntityMixin
-        range: string
+        range: date
         required: true
         inlined: true
         inlined_as_list: true
@@ -5067,7 +5078,7 @@ classes:
         domain_of:
         - DateStampedEntityMixin
         - APIDateStampedEntityMixin
-        range: string
+        range: date
         required: true
         inlined: true
         inlined_as_list: true
@@ -5103,7 +5114,7 @@ classes:
         recommended: true
         inlined: true
         inlined_as_list: true
-        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|pdb[0-9a-zA-Z]{4,8}))*$)
+        pattern: (^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)|(^(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8})(\s*,\s*(EMPIAR-[0-9]{5}|EMD-[0-9]{4,5}|PDB-[0-9a-zA-Z]{4,8}))*$)
       related_database_links:
         name: related_database_links
         description: Comma-separated list of related database links for the dataset.
@@ -5212,7 +5223,7 @@ classes:
         owner: AuthorMixin
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
       primary_author_status:
@@ -5226,7 +5237,7 @@ classes:
         owner: AuthorMixin
         domain_of:
         - AuthorMixin
-        range: string
+        range: boolean
         inlined: true
         inlined_as_list: true
 source_file: api/v2.0.0/api_models.yaml