add associations as attribute to datasets (#133)

* add associations as attribute to datasets * fix export tests
BioImage-Archive · Jul 31, 2024 · 634c699 · 634c699
1 parent e02f6a7
commit 634c699
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 43 deletions.
diff --git a/...t_data/experimental_imaging_datasets/S-BIADTEST/47a4ab60-c76d-4424-bfaa-c2a024de720c.json b/...t_data/experimental_imaging_datasets/S-BIADTEST/47a4ab60-c76d-4424-bfaa-c2a024de720c.json
@@ -2,6 +2,7 @@
     "title_id": "Study Component 1",
     "uuid": "47a4ab60-c76d-4424-bfaa-c2a024de720c",
     "description": "Description of study component 1",
+    "attribute": {},
     "analysis_method": [
         {
             "protocol_description": "Test image analysis",

diff --git a/bia-export/test/output_data/bia_export.json b/bia-export/test/output_data/bia_export.json
@@ -76,6 +76,7 @@
             "title_id": "Study Component 1",
             "uuid": "47a4ab60-c76d-4424-bfaa-c2a024de720c",
             "description": "Description of study component 1",
+            "attribute": {},
             "analysis_method": [
                 {
                     "protocol_description": "Test image analysis",

diff --git a/bia-export/test/test_local_convert.py b/bia-export/test/test_local_convert.py
@@ -18,4 +18,5 @@ def test_cli_export_website_studies(tmp_path):
 
 
     assert result.exit_code == 0
+    # Note this tests for exact equivance in files, i.e. order of fields and indentation matters
     assert filecmp.cmp(expected_output, outfile, shallow=False)
diff --git a/bia-ingest-shared-models/bia_ingest_sm/conversion/experimental_imaging_dataset.py b/bia-ingest-shared-models/bia_ingest_sm/conversion/experimental_imaging_dataset.py
@@ -59,6 +59,9 @@ def get_experimental_imaging_dataset(
         key_mapping = [
             ("image_analysis", "Image analysis", None,),
             ("image_correlation", "Image correlation", None,),
+            ("biosample", "Biosample", None,),
+            ("image_acquisition", "Image acquisition", None,),
+            ("specimen", "Specimen", None,),
         ]
         associations = get_generic_section_as_list(
             section, ["Associations",], key_mapping
@@ -96,6 +99,9 @@ def get_experimental_imaging_dataset(
             "analysis_method": analysis_method_list,
             "correlation_method": correlation_method_list,
             "example_image_uri": [],
+            "attribute": {
+                "associations": associations
+            }
         }
         model_dict["uuid"] = generate_experimental_imaging_dataset_uuid(model_dict)
 

diff --git a/bia-ingest-shared-models/test/utils.py b/bia-ingest-shared-models/test/utils.py
@@ -6,10 +6,8 @@
 
 from typing import Dict, List
 from bia_shared_datamodels import bia_data_model, semantic_models
-from bia_ingest_sm.conversion.utils import (
-    dict_to_uuid,
-    filter_model_dictionary
-)
+from bia_ingest_sm.conversion.utils import dict_to_uuid, filter_model_dictionary
+
 
 def get_test_annotation_method() -> List[bia_data_model.AnnotationMethod]:
     # For UUID
@@ -38,9 +36,15 @@ def get_test_annotation_method() -> List[bia_data_model.AnnotationMethod]:
 
     annotation_method = []
     for annotation_method_dict in annotation_method_info:
-        annotation_method_dict["uuid"] = dict_to_uuid(annotation_method_dict, attributes_to_consider)
-        annotation_method_dict = filter_model_dictionary(annotation_method_dict, bia_data_model.AnnotationMethod)
-        annotation_method.append(bia_data_model.AnnotationMethod.model_validate(annotation_method_dict))
+        annotation_method_dict["uuid"] = dict_to_uuid(
+            annotation_method_dict, attributes_to_consider
+        )
+        annotation_method_dict = filter_model_dictionary(
+            annotation_method_dict, bia_data_model.AnnotationMethod
+        )
+        annotation_method.append(
+            bia_data_model.AnnotationMethod.model_validate(annotation_method_dict)
+        )
     return annotation_method
 
 
@@ -70,7 +74,9 @@ def get_test_specimen_growth_protocol() -> List[bia_data_model.SpecimenGrowthPro
     protocol = []
     for protocol_dict in protocol_info:
         protocol_dict["uuid"] = dict_to_uuid(protocol_dict, attributes_to_consider)
-        protocol_dict = filter_model_dictionary(protocol_dict, bia_data_model.SpecimenGrowthProtocol)
+        protocol_dict = filter_model_dictionary(
+            protocol_dict, bia_data_model.SpecimenGrowthProtocol
+        )
         protocol.append(
             bia_data_model.SpecimenGrowthProtocol.model_validate(protocol_dict)
         )
@@ -107,9 +113,13 @@ def get_test_specimen_imaging_preparation_protocol() -> (
     protocol = []
     for protocol_dict in protocol_info:
         protocol_dict["uuid"] = dict_to_uuid(protocol_dict, attributes_to_consider)
-        protocol_dict = filter_model_dictionary(protocol_dict, bia_data_model.SpecimenImagingPrepartionProtocol)
+        protocol_dict = filter_model_dictionary(
+            protocol_dict, bia_data_model.SpecimenImagingPrepartionProtocol
+        )
         protocol.append(
-            bia_data_model.SpecimenImagingPrepartionProtocol.model_validate(protocol_dict)
+            bia_data_model.SpecimenImagingPrepartionProtocol.model_validate(
+                protocol_dict
+            )
         )
     return protocol
 
@@ -182,7 +192,9 @@ def get_test_biosample() -> List[bia_data_model.BioSample]:
     biosample = []
     for biosample_dict in biosample_info:
         biosample_dict["uuid"] = dict_to_uuid(biosample_dict, attributes_to_consider)
-        biosample_dict = filter_model_dictionary(biosample_dict, bia_data_model.BioSample)
+        biosample_dict = filter_model_dictionary(
+            biosample_dict, bia_data_model.BioSample
+        )
         biosample.append(bia_data_model.BioSample.model_validate(biosample_dict))
     return biosample
 
@@ -222,7 +234,9 @@ def get_test_image_acquisition() -> List[bia_data_model.ImageAcquisition]:
         image_acquisition_dict["uuid"] = dict_to_uuid(
             image_acquisition_dict, attributes_to_consider
         )
-        image_acquisition_dict = filter_model_dictionary(image_acquisition_dict, bia_data_model.ImageAcquisition)
+        image_acquisition_dict = filter_model_dictionary(
+            image_acquisition_dict, bia_data_model.ImageAcquisition
+        )
         image_acquisition.append(
             bia_data_model.ImageAcquisition.model_validate(image_acquisition_dict)
         )
@@ -272,6 +286,24 @@ def get_test_experimental_imaging_dataset() -> (
         ],
         "example_image_uri": [],
         "description": "Description of study component 1",
+        "attribute": {
+            "associations": [
+                {
+                    "image_analysis": "Test image analysis",
+                    "image_correlation": None,
+                    "biosample": "Test Biosample 1",
+                    "image_acquisition": "Test Primary Screen Image Acquisition",
+                    "specimen": "Test specimen 1",
+                },
+                {
+                    "image_analysis": "Test image analysis",
+                    "image_correlation": None,
+                    "biosample": "Test Biosample 2 ",
+                    "image_acquisition": "Test Secondary Screen Image Acquisition",
+                    "specimen": "Test specimen 1",
+                },
+            ]
+        },
     }
     experimental_imaging_dataset_uuid = dict_to_uuid(
         experimental_imaging_dataset_dict,
@@ -281,7 +313,9 @@ def get_test_experimental_imaging_dataset() -> (
         ],
     )
     experimental_imaging_dataset_dict["uuid"] = experimental_imaging_dataset_uuid
-    experimental_imaging_dataset_dict = filter_model_dictionary(experimental_imaging_dataset_dict, bia_data_model.ExperimentalImagingDataset)
+    experimental_imaging_dataset_dict = filter_model_dictionary(
+        experimental_imaging_dataset_dict, bia_data_model.ExperimentalImagingDataset
+    )
     experimental_imaging_dataset1 = (
         bia_data_model.ExperimentalImagingDataset.model_validate(
             experimental_imaging_dataset_dict
@@ -317,6 +351,17 @@ def get_test_experimental_imaging_dataset() -> (
         ],
         "example_image_uri": [],
         "description": "Description of study component 2",
+        "attribute": {
+            "associations": [
+                {
+                    "image_analysis": "Test image analysis",
+                    "image_correlation": None,
+                    "biosample": "Test Biosample 2 ",
+                    "image_acquisition": "Test Primary Screen Image Acquisition",
+                    "specimen": "Test specimen 2",
+                }
+            ]
+        },
     }
     experimental_imaging_dataset_uuid = dict_to_uuid(
         experimental_imaging_dataset_dict,

diff --git a/bia-shared-datamodels/src/bia_shared_datamodels/semantic_models.py b/bia-shared-datamodels/src/bia_shared_datamodels/semantic_models.py
@@ -218,6 +218,9 @@ class DatasetMixin(BaseModel):
     description: Optional[str] = Field(
         None, description="""Brief description of the dataset."""
     )
+    attribute: dict = Field(
+        description="""Freeform key-value pairs from user provided metadata (e.g. filelist data) and experimental fields."""
+    )
 
 
 class FileReference(BaseModel):

diff --git a/bia-shared-datamodels/test/utils.py b/bia-shared-datamodels/test/utils.py
@@ -106,9 +106,7 @@ def get_template_biosample() -> bia_data_model.BioSample:
     return biosample
 
 
-# Depends on:
-#   bia_data_model.BioSample
-#   bia_data_model.SpecimenImagingPreparationProtocol
+
 def get_template_specimen() -> bia_data_model.Specimen:
     specimen = bia_data_model.Specimen.model_validate(
         {
@@ -127,7 +125,6 @@ def get_template_specimen() -> bia_data_model.Specimen:
     return specimen
 
 
-# Depends on ExperimentalImagingDataset
 def get_template_annotation_method() -> bia_data_model.AnnotationMethod:
     annotation_method = bia_data_model.AnnotationMethod.model_validate(
         {
@@ -142,11 +139,7 @@ def get_template_annotation_method() -> bia_data_model.AnnotationMethod:
     return annotation_method
 
 
-# Depends on:
-#   bia_data_model.ExperimentalImagingDataset (circular dependency)
-#   bia_data_model.ImageAcquisition
-#   bia_data_model.ImageRepresentation
-#   bia_data_model.Specimen
+
 def get_template_experimentally_captured_image() -> (
     bia_data_model.ExperimentallyCapturedImage
 ):
@@ -161,10 +154,6 @@ def get_template_experimentally_captured_image() -> (
     )
 
 
-# Depends on:
-#   bia_data_model.ImageAnnotationDataset (circular dependency)
-#   bia_data_model.AnnotationMethod
-#   bia_data_model.ImageRepresentation
 def get_template_derived_image() -> bia_data_model.DerivedImage:
     derived_image = bia_data_model.DerivedImage.model_validate(
         {
@@ -182,15 +171,15 @@ def get_template_derived_image() -> bia_data_model.DerivedImage:
     return derived_image
 
 
-# Depends on:
-#   bia_data_model.AnnotationMethod
+
 def get_template_image_annotation_dataset() -> bia_data_model.ImageAnnotationDataset:
     image_annotation_dataset = bia_data_model.ImageAnnotationDataset.model_validate(
         {
             "uuid": uuid4(),
             "submitted_in_study_uuid": get_template_study().uuid,
             "title_id": "Template image annotation dataset",
             "example_image_uri": ["https://dummy.url.org"],
+            "attribute": {},
         }
     )
     return image_annotation_dataset
@@ -231,11 +220,6 @@ def get_template_image_correlation_method() -> semantic_models.ImageCorrelationM
     )
 
 
-# Depends on:
-#   bia_data_model.SpecimenPreparationProtocol
-#   bia_data_model.ImageAcquisition
-#   bia_data_model.BioSample
-#   bia_data_model.SpecimenGrowthProtocol
 def get_template_experimental_imaging_dataset() -> (
     bia_data_model.ExperimentalImagingDataset
 ):
@@ -252,15 +236,13 @@ def get_template_experimental_imaging_dataset() -> (
                     get_template_image_correlation_method().model_dump(),
                 ],
                 "example_image_uri": ["https://dummy.url.org"],
+                "attribute": {}
             }
         )
     )
     return experimental_imaging_dataset
 
 
-# Depends on:
-#   bia_data_model.ImageAnnotationDataset
-#   bia_data_model.ExperimentalImagingDataset (circular)
 def get_template_annotation_file_reference() -> bia_data_model.AnnotationFileReference:
     return bia_data_model.AnnotationFileReference.model_validate(
         {
@@ -281,11 +263,6 @@ def get_template_annotation_file_reference() -> bia_data_model.AnnotationFileRef
     )
 
 
-# Depends on:
-#   bia_data_model.ImageAnnotationDataset
-#   or
-#   bia_data_model.ExperimentalImagingDataset
-#   the latter is tested here.
 def get_template_file_reference() -> bia_data_model.FileReference:
     file_reference = bia_data_model.FileReference.model_validate(
         {
@@ -301,8 +278,7 @@ def get_template_file_reference() -> bia_data_model.FileReference:
     return file_reference
 
 
-# Depends on:
-#   bia_data_model.FileReference
+
 def get_template_image_representation() -> bia_data_model.ImageRepresentation:
     return bia_data_model.ImageRepresentation.model_validate(
         {
Original file line number	Diff line number	Diff line change
Expand Up		@@ -18,4 +18,5 @@ def test_cli_export_website_studies(tmp_path):


		assert result.exit_code == 0
		# Note this tests for exact equivance in files, i.e. order of fields and indentation matters
		assert filecmp.cmp(expected_output, outfile, shallow=False)