feat: add schema version to ingest config (#289)

* add schema version to ingest config
chanzuckerberg · Sep 25, 2024 · f8d2f2f · f8d2f2f
1 parent daff6b0
commit f8d2f2f
Show file tree

Hide file tree

Showing 327 changed files with 2,509 additions and 1,661 deletions.
diff --git a/ingestion_tools/dataset_configs/10000.yaml b/ingestion_tools/dataset_configs/10000.yaml
@@ -8,10 +8,6 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: pyTOM + Keras
-    method_links: &annotation_method_links
-      - link: https://github.com/ZauggGroup/DeePiCt
-        link_type: source_code
-        custom_name: DeePiCt on GitHub
     authors: &id001
     - ORCID: 0000-0002-9903-3667
       name: Sara Goetz
@@ -48,6 +44,10 @@ annotations:
       release_date: '2023-06-01'
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: &id003
+    - custom_name: DeePiCt on GitHub
+      link: https://github.com/ZauggGroup/DeePiCt
+      link_type: source_code
     method_type: hybrid
     version: 1.0
   sources:
@@ -65,11 +65,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: pyTOM + Keras
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -86,11 +86,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -107,11 +107,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -128,11 +128,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -149,11 +149,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -170,11 +170,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -191,11 +191,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -212,11 +212,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -233,11 +233,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -254,11 +254,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -275,11 +275,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -296,11 +296,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -317,11 +317,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: false
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -338,11 +338,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -364,7 +364,7 @@ dataset_keyphotos:
         thumbnail: https://www.ebi.ac.uk/pdbe/emdb-empiar/entryIcons/10988.gif
 datasets:
 - metadata:
-    authors: &id003
+    authors: &id004
     - ORCID: 0000-0002-4691-9501
       name: Irene de Teresa Trueba
       primary_author_status: true
@@ -397,7 +397,7 @@ datasets:
     cell_strain:
       id: NCBITaxon:284812
       name: Schizosaccharomyces pombe 972h-
-    cross_references: &dataset_cross_references
+    cross_references: &id005
       publications: doi:10.1101/2022.04.12.488077, doi:10.1038/s41592-022-01746-2
       related_database_entries: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
         EMD-14418, EMD-14419, EMD-14420
@@ -424,7 +424,8 @@ datasets:
       - '10000'
 depositions:
 - metadata:
-    authors: *id003
+    authors: *id004
+    cross_references: *id005
     dates: *id002
     deposition_description: Datasets comprising 20 annotated Schizosaccharomyces pombe
       cryo-electron tomograms, with 10 tomograms acquired using a Volta phase plate
@@ -437,7 +438,6 @@ depositions:
       localization across different species and acquisition conditions.
     deposition_identifier: 10000
     deposition_title: DeePiCt Training and Evaluation Datasets
-    cross_references: *dataset_cross_references
     deposition_types:
     - dataset
     - annotation
@@ -522,7 +522,7 @@ tomograms:
       - 0
       - 0
       - 1
-    authors: *id003
+    authors: *id004
     ctf_corrected: false
     fiducial_alignment_status: NON_FIDUCIAL
     offset:
@@ -538,6 +538,7 @@ tomograms:
   - source_glob:
       list_glob: tomograms/{run_name}.rec
       match_regex: .*\.rec
+version: 1.1.0
 voxel_spacings:
 - sources:
   - literal:

diff --git a/ingestion_tools/dataset_configs/10001.yaml b/ingestion_tools/dataset_configs/10001.yaml
@@ -8,10 +8,6 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: pyTOM + Keras
-    method_links: &annotation_method_links
-      - link: https://github.com/ZauggGroup/DeePiCt
-        link_type: source_code
-        custom_name: DeePiCt on GitHub
     authors: &id001
     - ORCID: 0000-0002-9903-3667
       name: Sara Goetz
@@ -48,6 +44,10 @@ annotations:
       release_date: '2023-06-01'
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: &id003
+    - custom_name: DeePiCt on GitHub
+      link: https://github.com/ZauggGroup/DeePiCt
+      link_type: source_code
     method_type: hybrid
     version: 1.0
   sources:
@@ -65,11 +65,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: pyTOM + Keras
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -326,11 +326,11 @@ annotations:
     annotation_publications: EMPIAR-10988, EMD-14412, EMD-14413, EMD-14415, EMD-14417,
       EMD-14418, EMD-14419, EMD-14420, 10.1101/2022.04.12.488077
     annotation_software: Keras + AMIRA
-    method_links: *annotation_method_links
     authors: *id001
     dates: *id002
     ground_truth_status: true
     is_curator_recommended: true
+    method_links: *id003
     method_type: hybrid
     version: 1.0
   sources:
@@ -352,7 +352,7 @@ dataset_keyphotos:
         thumbnail: https://www.ebi.ac.uk/pdbe/emdb-empiar/entryIcons/10988.gif
 datasets:
 - metadata:
-    authors: &id003
+    authors: &id004
     - ORCID: 0000-0002-4691-9501
       name: Irene de Teresa Trueba
       primary_author_status: true
@@ -492,7 +492,7 @@ tomograms:
       - 0
       - 0
       - 1
-    authors: *id003
+    authors: *id004
     ctf_corrected: false
     fiducial_alignment_status: NON_FIDUCIAL
     offset:
@@ -508,6 +508,7 @@ tomograms:
   - source_glob:
       list_glob: tomograms/{run_name}.rec
       match_regex: .*\.rec
+version: 1.1.0
 voxel_spacings:
 - sources:
   - literal:

diff --git a/ingestion_tools/dataset_configs/10002.yaml b/ingestion_tools/dataset_configs/10002.yaml
@@ -54,10 +54,6 @@ annotations:
       is_visualization_default: true
 - metadata:
     annotation_method: 2D CNN + 3D CNN hyperparameter tuning
-    method_links: &annotation_method_links
-      - link: https://github.com/ZauggGroup/DeePiCt
-        link_type: source_code
-        custom_name: DeePiCt on GitHub
     annotation_object:
       description: Actin DeePiCt training prediction
       id: GO:0005884
@@ -68,6 +64,10 @@ annotations:
     dates: *id002
     ground_truth_status: false
     is_curator_recommended: false
+    method_links:
+    - custom_name: DeePiCt on GitHub
+      link: https://github.com/ZauggGroup/DeePiCt
+      link_type: source_code
     method_type: automated
     version: 1.0
   sources:
@@ -246,6 +246,7 @@ tomograms:
   - source_glob:
       list_glob: tomograms/{run_name}_*.rec
       match_regex: .*
+version: 1.1.0
 voxel_spacings:
 - sources:
   - literal:

diff --git a/ingestion_tools/dataset_configs/10003.yaml b/ingestion_tools/dataset_configs/10003.yaml
@@ -64,7 +64,7 @@ datasets:
     cell_strain:
       id: NCBITaxon:272634
       name: Mycoplasmoides pneumoniae M129
-    cross_references: &dataset_cross_references
+    cross_references: &id003
       publications: doi:10.1038/s41586-022-05255-2, doi:10.1038/s41592-020-01054-7
       related_database_entries: EMPIAR-10731, EMPIAR-10499
     dataset_description: Cryo-electron tomograms of native M. pneumoniae cells. Cells
@@ -94,12 +94,14 @@ datasets:
 depositions:
 - metadata:
     authors: *id002
+    cross_references: *id003
     dates: *id001
-    deposition_description: Cryo-electron tomograms of native M. pneumoniae cells. Cells
-      treated with chloramphenicol. Annotations of 70S ribosome refined using RELION and M.
+    deposition_description: Cryo-electron tomograms of native M. pneumoniae cells.
+      Cells treated with chloramphenicol. Annotations of 70S ribosome refined using
+      RELION and M.
     deposition_identifier: 10003
-    deposition_title: Warp/M evaluation dataset - Mycoplasma pneumoniae treated with chloramphenicol
-    cross_references: *dataset_cross_references
+    deposition_title: Warp/M evaluation dataset - Mycoplasma pneumoniae treated with
+      chloramphenicol
     deposition_types:
     - dataset
     - annotation
@@ -193,6 +195,7 @@ tomograms:
   - source_glob:
       list_glob: reconstructions/{mapped_tomo_name}*.mrc
       match_regex: .*
+version: 1.1.0
 voxel_spacings:
 - sources:
   - literal: