fix tests after model update with version and model version, and exte…

…nd forbidding extra fields to nested nodes (#135)

* fix tests after model update with version and model version, and extend forbidding extra fields to nested nodes

* Update experimental_imaging_dataset.py

* Update utils.py

.dockerignore

@@ -0,0 +1,4 @@
+**/.share
+**/.git
+**/.venv
+.venv

api/.vscode/settings.json

@@ -4,7 +4,7 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
-    "python.testing.cwd": "${workspaceFolder}",
+    "python.testing.cwd": "${workspaceFolder}/api",
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter",
         "editor.formatOnSave": true

api/Dockerfile

@@ -3,27 +3,23 @@ FROM python:3.10.0
 
 EXPOSE 8080
 
-
-# Override default shell and use bash and bia env
-#RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
-#    /bin/bash ~/miniconda.sh -b -p /opt/conda
-#ENV PATH=/opt/conda/bin:$PATH
-#RUN conda env create -f conda_env.yml
-
 RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH=/root/.local/bin:$PATH
 
 # add files that poetry needs first so layers up to (including) pulling dependencies get reused
+WORKDIR /bia-integrator
+
 #   only add poetry.lock if it exists (building on local)
-ADD ./poetry.lock* /integrator-api/poetry.lock
-ADD ./pyproject.toml /integrator-api/pyproject.toml
-WORKDIR /integrator-api
+ADD ./api/poetry.lock* api/poetry.lock
+ADD ./api/pyproject.toml api/pyproject.toml
+ADD ./bia-shared-datamodels bia-shared-datamodels
 
+WORKDIR /bia-integrator/api
 RUN poetry install
 
 # Everything up to here should be reused most times
 
 # add the actual project, which is what is often changed in between two different container builds
-ADD ./ /integrator-api
+ADD ./api ./
 
-CMD ["poetry", "run", "uvicorn", "--workers", "4", "--port", "8080", "--log-config", "./src/log_config.yml", "--host", "0.0.0.0", "src.app:app"]
+CMD ["poetry", "run", "uvicorn", "--workers", "4", "--port", "8080", "--log-config", "./src/log_config.yml", "--host", "0.0.0.0", "api.app:app"]

api/docker-compose.yml

@@ -20,20 +20,19 @@ services:
       #     #- "./.share/keyfile.secret:/home/share/keyfile/keyfile.secret"
   bia-integrator-api:
     build:
-      context: .
+      context: ..
+      dockerfile: ./api/Dockerfile
+    env_file: ./.env_compose
     container_name: api
     hostname: api
     ports:
       - 8080:8080
     expose:
       - 8080
-    volumes:
-      - "./.env_compose:/integrator-api/.env"
     depends_on:
       - biaint-mongo
 
-
 volumes:
   api_data:
     #? do we always want a 'seed db'?
-    #external: true
+    #external: true

api/pyproject.toml

@@ -23,6 +23,8 @@ ome-types = "^0.4.2"
 xsdata = "23.8"
 lxml = "^4.9.3"
 json-log-formatter = "^1.0"
+#@TODO: CHANGEME
+bia-shared-datamodels = { path = "../../prs/bia-integrator/bia-shared-datamodels", develop = true }
 
 [tool.poetry.group.dev.dependencies]
 locust = "^2.16.1"

bia-export/test/input_data/biosample/S-BIADTEST/64a67727-4e7c-469a-91c4-6219ae072e99.json

@@ -17,5 +17,6 @@
     ],
     "intrinsic_variable_description": [
         "Test intrinsic variable 1\\nwith escaped character"
-    ]
+    ],
+    "version": 1
 }

bia-export/test/input_data/biosample/S-BIADTEST/6950718c-4917-47a1-a807-11b874e80a23.json

@@ -17,5 +17,6 @@
     ],
     "intrinsic_variable_description": [
         "Test intrinsic variable 2"
-    ]
+    ],
+    "version": 1
 }

bia-export/test/input_data/experimental_imaging_datasets/S-BIADTEST/47a4ab60-c76d-4424-bfaa-c2a024de720c.json

@@ -11,5 +11,6 @@
     ],
     "submitted_in_study_uuid": "a2fdbd58-ee11-4cd9-bc6a-f3d3da7fff71",
     "correlation_method": [],
-    "example_image_uri": []
+    "example_image_uri": [],
+    "version": 1
 }

bia-export/test/input_data/image_acquisitions/S-BIADTEST/c2e44a1b-a43c-476e-8ddf-8587f4c955b3.json

@@ -4,5 +4,6 @@
     "protocol_description": "Test image acquisition parameters 1",
     "imaging_instrument_description": "Test imaging instrument 1",
     "fbbi_id": [],
-    "imaging_method_name": "confocal microscopy"
+    "imaging_method_name": "confocal microscopy",
+    "version": 1
 }

bia-export/test/input_data/specimen_imaging_preparation_protocols/S-BIADTEST/7199d730-29f1-4ad8-b599-e9089cbb2d7b.json

@@ -2,5 +2,6 @@
     "title_id": "Test specimen 1",
     "uuid": "7199d730-29f1-4ad8-b599-e9089cbb2d7b",
     "protocol_description": "Test sample preparation protocol 1",
-    "signal_channel_information": []
+    "signal_channel_information": [],
+    "version": 1
 }

bia-export/test/input_data/studies/S-BIADTEST.json

@@ -1,5 +1,10 @@
 {
     "uuid": "a2fdbd58-ee11-4cd9-bc6a-f3d3da7fff71",
+    "version": 1,
+    "model": {
+        "type_name": "Study",
+        "version": 1
+    },
     "accession_id": "S-BIADTEST",
     "licence": "CC0",
     "author": [

bia-export/test/output_data/bia_export.json

@@ -1,5 +1,10 @@
 {
     "uuid": "a2fdbd58-ee11-4cd9-bc6a-f3d3da7fff71",
+    "version": 1,
+    "model": {
+        "type_name": "Study",
+        "version": 1
+    },
     "accession_id": "S-BIADTEST",
     "licence": "CC0",
     "author": [
@@ -75,6 +80,11 @@
         {
             "title_id": "Study Component 1",
             "uuid": "47a4ab60-c76d-4424-bfaa-c2a024de720c",
+            "version": 1,
+            "model": {
+                "type_name": "ExperimentalImagingDataset",
+                "version": 1
+            },
             "description": "Description of study component 1",
             "attribute": {},
             "analysis_method": [

bia-ingest-shared-models/bia_ingest_sm/conversion/annotation_method.py

@@ -53,6 +53,7 @@ def extract_annotation_method_dicts(submission: Submission) -> List[Dict[str, An
         model_dict["accno"] = section.__dict__.get("accno", "")
         model_dict["accession_id"] = submission.accno
         model_dict["uuid"] = generate_annotation_method_uuid(model_dict)
+        model_dict["version"] = 1
         model_dict = filter_model_dictionary(model_dict, bia_data_model.AnnotationMethod)
 
         model_dicts.append(model_dict)

bia-ingest-shared-models/bia_ingest_sm/conversion/biosample.py

@@ -75,6 +75,7 @@ def extract_biosample_dicts(submission: Submission) -> List[Dict[str, Any]]:
 
         model_dict["accession_id"] = submission.accno
         model_dict["uuid"] = generate_biosample_uuid(model_dict)
+        model_dict["version"] = 1
         model_dict = filter_model_dictionary(model_dict, bia_data_model.BioSample)
         model_dicts.append(model_dict)
 

bia-ingest-shared-models/bia_ingest_sm/conversion/experimental_imaging_dataset.py

@@ -99,6 +99,7 @@ def get_experimental_imaging_dataset(
             "analysis_method": analysis_method_list,
             "correlation_method": correlation_method_list,
             "example_image_uri": [],
+            "version": 1,
             "attribute": {
                 "associations": associations
             }
@@ -140,4 +141,4 @@ def generate_experimental_imaging_dataset_uuid(experimental_imaging_dataset_dict
         "title_id", 
         "submitted_in_study_uuid",
     ]
-    return dict_to_uuid(experimental_imaging_dataset_dict, attributes_to_consider)
+    return dict_to_uuid(experimental_imaging_dataset_dict, attributes_to_consider)

bia-ingest-shared-models/bia_ingest_sm/conversion/file_reference.py

@@ -45,6 +45,7 @@ def get_file_reference_by_study_component(
                 "accession_id": submission.accno,
                 "file_path": str(f.path),
                 "size_in_bytes": str(f.size),
+                "version": 1
             }
             fileref_uuid = dict_to_uuid(
                 file_dict, ["accession_id", "file_path", "size_in_bytes"]

bia-ingest-shared-models/bia_ingest_sm/conversion/image_acquisition.py

@@ -52,6 +52,7 @@ def extract_image_acquisition_dicts(submission: Submission) -> List[Dict[str, An
         model_dict["accno"] = section.__dict__.get("accno", "")
         model_dict["accession_id"] = submission.accno
         model_dict["uuid"] = generate_image_acquisition_uuid(model_dict)
+        model_dict["version"] = 1
         model_dict = filter_model_dictionary(model_dict, bia_data_model.ImageAcquisition)
         model_dicts.append(model_dict)
 

bia-ingest-shared-models/bia_ingest_sm/conversion/specimen_growth_protocol.py

@@ -47,6 +47,7 @@ def extract_specimen_growth_protocol_dicts(submission: Submission) -> List[Dict[
         model_dict["accno"] = section.__dict__.get("accno", "")
         model_dict["accession_id"] = submission.accno
         model_dict["uuid"] = generate_specimen_growth_protocol_uuid(model_dict)
+        model_dict["version"] = 1
         model_dict = filter_model_dictionary(model_dict, bia_data_model.SpecimenGrowthProtocol)
 
         model_dicts.append(model_dict)

bia-ingest-shared-models/bia_ingest_sm/conversion/specimen_imaging_preparation_protocol.py

@@ -50,6 +50,7 @@ def extract_specimen_preparation_protocol_dicts(submission: Submission) -> List[
         model_dict["accno"] = section.__dict__.get("accno", "")
         model_dict["accession_id"] = submission.accno
         model_dict["uuid"] = generate_specimen_imaging_preparation_uuid(model_dict)
+        model_dict["version"] = 1
         model_dict = filter_model_dictionary(model_dict, bia_data_model.SpecimenImagingPrepartionProtocol)
 
         model_dicts.append(model_dict)

bia-ingest-shared-models/bia_ingest_sm/conversion/study.py

@@ -65,6 +65,7 @@ def get_study(
         "author": [c.model_dump() for c in contributors],
         "grant": [g.model_dump() for g in grants],
         "attribute": study_attributes,
+        "version": 1
     }
     # study_uuid = dict_to_uuid(study_dict, ["accession_id",])
     # study_dict["uuid"] = study_uuid

bia-ingest-shared-models/bia_ingest_sm/conversion/utils.py

@@ -155,5 +155,5 @@ def persist(object_list: List, object_path: str, sumbission_accno: str):
 
 def filter_model_dictionary(dictionary: dict, target_model: Type[BaseModel]):
     accepted_fields = target_model.model_fields.keys()
-    result_dict = {key: dictionary[key] for key in accepted_fields}
+    result_dict = {key: dictionary[key]  for key in accepted_fields if key in dictionary}
     return result_dict

bia-ingest-shared-models/test/utils.py

@@ -31,6 +31,7 @@ def get_test_annotation_method() -> List[bia_data_model.AnnotationMethod]:
             "annotation_coverage": "",
             "method_type": "other",
             "source_dataset": [],
+            "version": 1,
         },
     ]
 
@@ -62,12 +63,14 @@ def get_test_specimen_growth_protocol() -> List[bia_data_model.SpecimenGrowthPro
             "accession_id": "S-BIADTEST",
             "title_id": "Test specimen 1",
             "protocol_description": "Test growth protocol 1",
+            "version": 1,
         },
         {
             "accno": "Specimen-2",
             "accession_id": "S-BIADTEST",
             "title_id": "Test specimen 2",
             "protocol_description": "Test growth protocol 2",
+            "version": 1,
         },
     ]
 
@@ -100,13 +103,15 @@ def get_test_specimen_imaging_preparation_protocol() -> (
             "title_id": "Test specimen 1",
             "protocol_description": "Test sample preparation protocol 1",
             "signal_channel_information": [],
+            "version": 1,
         },
         {
             "accno": "Specimen-2",
             "accession_id": "S-BIADTEST",
             "title_id": "Test specimen 2",
             "protocol_description": "Test sample preparation protocol 2",
             "signal_channel_information": [],
+            "version": 1,
         },
     ]
 
@@ -168,6 +173,7 @@ def get_test_biosample() -> List[bia_data_model.BioSample]:
             "intrinsic_variable_description": [
                 "Test intrinsic variable 1\nwith escaped character",
             ],
+            "version": 1,
         },
         {
             "accno": "Biosample-2",
@@ -186,6 +192,7 @@ def get_test_biosample() -> List[bia_data_model.BioSample]:
             "intrinsic_variable_description": [
                 "Test intrinsic variable 2",
             ],
+            "version": 1,
         },
     ]
 
@@ -218,6 +225,7 @@ def get_test_image_acquisition() -> List[bia_data_model.ImageAcquisition]:
             "imaging_instrument_description": "Test imaging instrument 1",
             "imaging_method_name": "confocal microscopy",
             "fbbi_id": [],
+            "version": 1,
         },
         {
             "accno": "Image acquisition-7",
@@ -227,6 +235,7 @@ def get_test_image_acquisition() -> List[bia_data_model.ImageAcquisition]:
             "imaging_instrument_description": "Test imaging instrument 2",
             "imaging_method_name": "fluorescence microscopy",
             "fbbi_id": [],
+            "version": 1,
         },
     ]
     image_acquisition = []
@@ -286,6 +295,7 @@ def get_test_experimental_imaging_dataset() -> (
         ],
         "example_image_uri": [],
         "description": "Description of study component 1",
+        "version": 1,
         "attribute": {
             "associations": [
                 {
@@ -328,16 +338,19 @@ def get_test_experimental_imaging_dataset() -> (
             "accession_id": "S-BIADTEST",
             "file_path": "study_component2/im06.png",
             "size_in_bytes": 3,
+            "version": 1,
         },
         {
             "accession_id": "S-BIADTEST",
             "file_path": "study_component2/im08.png",
             "size_in_bytes": 123,
+            "version": 1,
         },
         {
             "accession_id": "S-BIADTEST",
             "file_path": "study_component2/ann01-05",
             "size_in_bytes": 11,
+            "version": 1,
         },
     ]
     experimental_imaging_dataset_dict = {
@@ -351,6 +364,7 @@ def get_test_experimental_imaging_dataset() -> (
         ],
         "example_image_uri": [],
         "description": "Description of study component 2",
+        "version": 1,
         "attribute": {
             "associations": [
                 {
@@ -540,6 +554,7 @@ def get_test_study() -> bia_data_model.Study:
             "Test keyword3",
         ],
         "grant": [g.model_dump() for g in grant],
+        "version": 1,
     }
     study_uuid = dict_to_uuid(
         study_dict,

bia-shared-datamodels/src/bia_shared_datamodels/bia_data_model.py

@@ -25,9 +25,6 @@ class DocumentMixin(BaseModel):
         Optional because for some usecases (e.g. api) we want to accept objects without it because we have the info we need to set it."""
     )
 
-    # Throw error if you try to validate/create model from a dictionary with keys that aren't a field in the model
-    model_config = ConfigDict(extra="forbid")
-
     def __init__(self, *args, **data):
         model_version_spec = self.model_config.get("model_version_latest")
         if model_version_spec is None:
@@ -41,6 +38,7 @@ def __init__(self, *args, **data):
         )
         model_metadata_existing = data.get("model", None)
         if model_metadata_existing:
+            model_metadata_existing = ModelMetadata(**model_metadata_existing)
             if model_metadata_existing != model_metadata_expected:
                 raise exceptions.UnexpectedDocumentType(
                     f"Document {str(data.get('uuid'))} has model metadata {model_metadata_existing}, expected : {model_metadata_expected}"