Assign image (#262)

* Initial commit - save WIP

* Updated functions for creating uuids of image related objects

* Save WIP - functions to create Image, Specimen and CreationProcess

* Save WIP at point of cli creation

* Save alpha version. Assign to Image and create ImageReperesentation work

Save initial version with assigning to Image and creating ImageRepresentation.
Need to:
* Improve structure of bia_assign_image package
    - Decide on whether to use PersistenceStrategy of bia_ingest
    - Decide on where to create ImageRepresentation
    - Names of modules
    - Logging
* Move tests.mock_objects to bia_test_data
* Make tests for Image with more than 1 file reference

* Fix bug in cli.py

* Change image representation uuid generation to use value of enums

* Add test version of migration script

* Use settings in cli and tests

* Allow use of public and private apis in persister

* Save changes after to allow read from api but persist to disk - needed for migration

* Correct error with input_image_uuid in creation_process

* Fix minor bug and add script for updating example image uri

* Allow migrate image representation scripts to work for all accession ids

* Remove ability to fetch and persist to different sources

* Sav WIP after changes to address Draft PR comments

* Rebase with current main and modify tests as necessary

* Update readme files

* Modify readmes and python version as suggested in PR review

bia-assign-image/README.md

@@ -0,0 +1,63 @@
+## Description
+This sub-package assigns file reference(s) to BIA Image objects and creates image representations but *not* actual images associated with the representations. It is in alpha mode!
+
+## Setup
+
+1. Install the project using poetry.
+2. All artefacts are currently written to a hard coded dir ~/.cache/bia-integrator-data-sm
+3. Only `--persistence-mode disk`) works at the moment
+
+## Usage
+This package has 2 cli applications:
+ * **assign**: used to assign file reference(s) to BIA Image objects
+ * **representations**: used to create image representation objects (without conversion of images) from BIA Image objects.
+
+## Assigning file refernce(s) to BIA Image objects
+To create a BIA Image of a set of file references run:
+``` sh
+poetry run bia-assign-image assign <STUDY ACCESSION ID> <LIST OF FILE REFERENCE UUIDS>
+```
+E.g. Assuming the study S-BIAD1285 has been ingested:
+```
+poetry run bia-assign-image assign S-BIAD1285 b768fb72-7ea2-4b80-b54d-bdf5ca280bfd
+```
+By default this creates BIA Image objects under:
+~/.cache/bia-integrator-data-sm/
+  image/
+    image_1_uuid.json
+    ...
+
+## Creating representations (without conversion of images)
+To create Image representations and experimentally captured images from file references (without image conversion also occuring), run:
+``` sh
+$ poetry run bia-assign-image representations create <STUDY ACCESSION ID> <IMAGE UUID>
+```
+E.g. Assuming the command above has been run:
+```sh
+$ poetry run bia-assign-image representations create S-BIAD1285 92fd093d-c8d2-4d89-ba28-9a9891cec73f
+```
+
+By default this create ImageRepresentation objects locally under:
+```sh
+  image_representation/
+    image_representation_1_uuid.json
+    image_representation_2_uuid.json
+    image_representation_3_uuid.json
+    ...
+```
+
+By default this creates 3 image representations (but not the actual images) for each of the file references.
+1. UPLOADED_BY_SUBMITTER
+2. INTERACTIVE_DISPLAY (ome_zarr)
+3. THUMBNAIL
+
+The STATIC_DISPLAY representation is not created by default because the BIA website only needs one static display per experimental imaging dataset. All interactive images need a thumbnail for the website, so they are usually created together.
+
+An option can be passed into the command to specify representations to create. E.g. to create only THUMBNAIL and STATIC_DISPLAY:
+```sh
+$ poetry run bia-assign-image representations create --reps-to-create THUMBNAIL --reps-to-create STATIC_DISPLAY S-BIAD1285 92fd093d-c8d2-4d89-ba28-9a9891cec73f
+```
+
+## Scripts to migrate artefacts from API models used in SAB to API models as of 12/12/2024
+
+See [scripts/README.md](scripts/README.md)

bia-assign-image/bia_assign_image/cli.py

@@ -0,0 +1,250 @@
+from typing import List, Any
+from typing import Annotated
+import typer
+from bia_shared_datamodels import bia_data_model, uuid_creation, semantic_models
+from bia_shared_datamodels.semantic_models import ImageRepresentationUseType
+from bia_ingest.persistence_strategy import (
+    PersistenceMode,
+    persistence_strategy_factory,
+)
+from bia_assign_image import (
+    image,
+    specimen,
+    creation_process,
+)
+from bia_assign_image.image_representation import get_image_representation
+from bia_assign_image.config import settings, api_client
+
+# For read only client
+
+import logging
+
+app = typer.Typer()
+representations_app = typer.Typer()
+app.add_typer(
+    representations_app,
+    name="representations",
+    help="Create specified representations",
+)
+
+logging.basicConfig(
+    #    level=logging.INFO, format="%(message)s", handlers=[RichHandler(show_time=False)]
+    level=logging.INFO,
+    format="%(message)s",
+)
+
+logger = logging.getLogger()
+
+
+def _get_value_from_attribute_list(
+    attribute_list: List[semantic_models.Attribute],
+    attribute_name: str,
+    default: Any = [],
+) -> Any:
+    """Get the value of named attribute from a list of attributes"""
+
+    # Assumes attribute.value is a Dict
+    return next(
+        (
+            attribute.value[attribute_name]
+            for attribute in attribute_list
+            if attribute.name == attribute_name
+        ),
+        default,
+    )
+
+
+@app.command(help="Assign listed file references to an image")
+def assign(
+    accession_id: Annotated[str, typer.Argument()],
+    file_reference_uuids: Annotated[List[str], typer.Argument()],
+    persistence_mode: Annotated[
+        PersistenceMode, typer.Option(case_sensitive=False)
+    ] = PersistenceMode.disk,
+    dryrun: Annotated[bool, typer.Option()] = False,
+) -> None:
+    persister = persistence_strategy_factory(
+        persistence_mode,
+        output_dir_base=settings.bia_data_dir,
+        accession_id=accession_id,
+        api_client=api_client,
+    )
+
+    file_reference_uuid_list = file_reference_uuids[0].split(" ")
+    file_references = persister.fetch_by_uuid(
+        file_reference_uuid_list, bia_data_model.FileReference
+    )
+    dataset_uuids = [f.submission_dataset_uuid for f in file_references]
+    assert len(set(dataset_uuids)) == 1
+    submission_dataset_uuid = dataset_uuids[0]
+    dataset = persister.fetch_by_uuid(
+        [
+            submission_dataset_uuid,
+        ],
+        bia_data_model.Dataset,
+    )[0]
+
+    image_uuid = uuid_creation.create_image_uuid(file_reference_uuid_list)
+
+    image_acquisition_protocol_uuid = _get_value_from_attribute_list(
+        dataset.attribute, "image_acquisition_protocol_uuid"
+    )
+    image_preparation_protocol_uuid = _get_value_from_attribute_list(
+        dataset.attribute, "specimen_imaging_preparation_protocol_uuid"
+    )
+    bio_sample_uuid = _get_value_from_attribute_list(
+        dataset.attribute, "bio_sample_uuid"
+    )
+    image_pre_requisites = [
+        len(image_acquisition_protocol_uuid),
+        len(image_preparation_protocol_uuid),
+        len(bio_sample_uuid),
+    ]
+
+    assert_error_msg = (
+        "Incomplete requisites for creating Specimen AND CreationProcess. "
+        + "Need ImageAcquisitionProtocol, SpecimenImagePreparationProtocol and BioSample UUIDs in "
+        + "dataset attributes. Got "
+        + f"ImageAcquisitionProtocol: {image_acquisition_protocol_uuid},"
+        + f"SpecimenImagePreparationProtocol: {image_preparation_protocol_uuid},"
+        + f"BioSample: {bio_sample_uuid}"
+    )
+    assert any(image_pre_requisites) and all(image_pre_requisites), assert_error_msg
+
+    if not any(image_pre_requisites):
+        logger.warning(
+            "No image_preparation_protocol or bio_sample uuids found in dataset attributes. No Specimen object will be created for this image!"
+        )
+        bia_specimen = None
+    else:
+        bia_specimen = specimen.get_specimen(
+            image_uuid, image_preparation_protocol_uuid, bio_sample_uuid
+        )
+        if dryrun:
+            logger.info(
+                f"Dryrun: Created specimen(s) {bia_specimen}, but not persisting."
+            )
+        else:
+            persister.persist(
+                [
+                    bia_specimen,
+                ]
+            )
+            logger.info(
+                f"Generated bia_data_model.Specimen object {bia_specimen.uuid} and persisted to {persistence_mode}"
+            )
+
+    if not bia_specimen:
+        logger.warning("Creating CreationProcess with no Specimen")
+    if not image_acquisition_protocol_uuid:
+        logger.warning("Creating CreationProcess with no ImageAcquisitionProtocol")
+    bia_creation_process = creation_process.get_creation_process(
+        image_uuid,
+        bia_specimen.uuid,
+        image_acquisition_protocol_uuid,
+    )
+    if dryrun:
+        logger.info(
+            f"Dryrun: Created creation process(es) {bia_creation_process}, but not persisting."
+        )
+    else:
+        persister.persist(
+            [
+                bia_creation_process,
+            ]
+        )
+        logger.info(
+            f"Generated bia_data_model.CreationProcess object {bia_creation_process.uuid} and persisted to {persistence_mode}"
+        )
+
+    bia_image = image.get_image(
+        submission_dataset_uuid,
+        bia_creation_process.uuid,
+        file_references=file_references,
+    )
+    if dryrun:
+        logger.info(f"Dryrun: Created Image(s) {bia_image}, but not persisting.")
+    else:
+        persister.persist(
+            [
+                bia_image,
+            ]
+        )
+        logger.info(
+            f"Generated bia_data_model.Image object {bia_image.uuid} and persisted to {persistence_mode}"
+        )
+
+
+@representations_app.command(help="Create specified representations")
+def create(
+    accession_id: Annotated[str, typer.Argument()],
+    image_uuid_list: Annotated[List[str], typer.Argument()],
+    persistence_mode: Annotated[
+        PersistenceMode, typer.Option(case_sensitive=False)
+    ] = PersistenceMode.disk,
+    reps_to_create: Annotated[
+        List[ImageRepresentationUseType], typer.Option(case_sensitive=False)
+    ] = [
+        ImageRepresentationUseType.UPLOADED_BY_SUBMITTER,
+        ImageRepresentationUseType.THUMBNAIL,
+        ImageRepresentationUseType.INTERACTIVE_DISPLAY,
+    ],
+    dryrun: Annotated[bool, typer.Option()] = False,
+    verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False,
+) -> None:
+    """Create representations for specified file reference(s)"""
+
+    if verbose:
+        logger.setLevel(logging.DEBUG)
+
+    persister = persistence_strategy_factory(
+        persistence_mode,
+        output_dir_base=settings.bia_data_dir,
+        accession_id=accession_id,
+        api_client=api_client,
+    )
+
+    bia_images = persister.fetch_by_uuid(image_uuid_list, bia_data_model.Image)
+    for bia_image in bia_images:
+        file_references = persister.fetch_by_uuid(
+            bia_image.original_file_reference_uuid, bia_data_model.FileReference
+        )
+        for representation_use_type in reps_to_create:
+            logger.debug(
+                f"starting creation of image representation of use type {representation_use_type.value} for Image {bia_image.uuid}"
+            )
+            image_representation = get_image_representation(
+                accession_id,
+                file_references,
+                bia_image,
+                use_type=representation_use_type,
+            )
+            if image_representation:
+                message = f"COMPLETED: Creation of image representation {image_representation.uuid} of use type {representation_use_type.value} for bia_data_model.Image {bia_image.uuid} of {accession_id}"
+                logger.info(message)
+                if dryrun:
+                    logger.info(
+                        f"Not persisting image representation:{image_representation}."
+                    )
+                else:
+                    persister.persist(
+                        [
+                            image_representation,
+                        ]
+                    )
+                    logger.info(
+                        f"Persisted image_representation {image_representation.uuid}"
+                    )
+
+            else:
+                message = f"WARNING: Could NOT create image representation of use type {representation_use_type.value} for bia_data_model.Image {bia_image.uuid} of {accession_id}"
+                logger.warning(message)
+
+
+@app.callback()
+def main() -> None:
+    return
+
+
+if __name__ == "__main__":
+    app()

bia-assign-image/bia_assign_image/config.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+import os
+import logging
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from bia_integrator_api.util import get_client_private
+
+logger = logging.getLogger("__main__." + __name__)
+
+default_output_base = (
+    f"{Path(os.environ.get('HOME', '')) / '.cache' / 'bia-integrator-data-sm'}"
+)
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=f"{Path(__file__).parent.parent / '.env'}",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        # extra="forbid",
+    )
+
+    bia_data_dir: str = Field(default_output_base)
+    endpoint_url: str = Field("https://uk1s3.embassy.ebi.ac.uk")
+    bucket_name: str = Field("bia-integrator-data")
+    cache_root_dirpath: Path = Field(Path.home() / ".cache" / "bia-converter")
+    bia_api_basepath: str = Field(
+        "http://localhost:8080", json_schema_extra={"env": "BIA_API_BASEPATH"}
+    )
+    bia_api_username: str = Field(
+        "[email protected]", json_schema_extra={"env": "BIA_API_USERNAME"}
+    )
+    bia_api_password: str = Field("test", json_schema_extra={"env": "BIA_API_PASSWORD"})
+
+
+settings = Settings()
+
+try:
+    api_client = get_client_private(
+        username=settings.bia_api_username,
+        password=settings.bia_api_password,
+        api_base_url=settings.bia_api_basepath,
+    )
+except Exception as e:
+    message = f"Could not initialise api_client: {e}"
+    logger.warning(message)
+    api_client = None

bia-assign-image/bia_assign_image/creation_process.py

@@ -0,0 +1,23 @@
+from uuid import UUID
+from typing import Optional, List
+from bia_shared_datamodels import bia_data_model, uuid_creation
+
+
+def get_creation_process(
+    output_image_uuid: UUID,
+    subject_specimen_uuid: Optional[UUID | None] = None,
+    image_acquisition_protocol_uuid: Optional[List[UUID] | List] = [],
+    input_image_uuid: Optional[List[UUID] | List] = [],
+) -> bia_data_model.CreationProcess:
+    model_dict = {
+        "version": 0,
+        "uuid": uuid_creation.create_creation_process_uuid(output_image_uuid),
+    }
+    if subject_specimen_uuid:
+        model_dict["subject_specimen_uuid"] = subject_specimen_uuid
+    if image_acquisition_protocol_uuid:
+        model_dict["image_acquisition_protocol_uuid"] = image_acquisition_protocol_uuid
+    if input_image_uuid:
+        model_dict["input_image_uuid"] = input_image_uuid
+
+    return bia_data_model.CreationProcess.model_validate(model_dict)