Updating tomogram id generation

ingestion_tools/scripts/importers/alignment.py

@@ -17,7 +17,7 @@
 class AlignmentIdentifierHelper(IdentifierHelper):
     @classmethod
     def _get_container_key(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:
-        return parents["run"].get_output_path()
+        return "-".join(["alignment", parents["run"].get_output_path()])
 
     @classmethod
     def _get_metadata_glob(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:

ingestion_tools/scripts/importers/annotation.py

@@ -20,7 +20,7 @@
 class AnnotationIdentifierHelper(IdentifierHelper):
     @classmethod
     def _get_container_key(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs):
-        return parents["voxel_spacing"].get_output_path()
+        return "-".join(["annotation", parents["voxel_spacing"].get_output_path()])
 
     @classmethod
     def _get_metadata_glob(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:

ingestion_tools/scripts/importers/tiltseries.py

@@ -12,7 +12,7 @@
 class TiltSeriesIdentifierHelper(IdentifierHelper):
     @classmethod
     def _get_container_key(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:
-        return parents["run"].get_output_path()
+        return "-".join(["tiltseries", parents["run"].get_output_path()])
 
     @classmethod
     def _get_metadata_glob(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:

ingestion_tools/scripts/importers/tomogram.py

@@ -1,4 +1,5 @@
 import os
+import re
 from typing import Any
 
 from common.config import DepositionImportConfig
@@ -14,13 +15,14 @@
 class TomogramIdentifierHelper(IdentifierHelper):
     @classmethod
     def _get_container_key(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:
-        return parents["voxel_spacing"].get_output_path()
+        return "-".join(["tomogram", parents["run"].get_output_path()])
 
     @classmethod
     def _get_metadata_glob(cls, config: DepositionImportConfig, parents: dict[str, Any], *args, **kwargs) -> str:
         voxel_spacing = parents["voxel_spacing"]
         tomogram_dir_path = config.resolve_output_path("tomogram", voxel_spacing)
-        return os.path.join(tomogram_dir_path, "*tomogram_metadata.json")
+        search_path = re.sub(r"/VoxelSpacing[0-9.]+/", "/VoxelSpacing*/", tomogram_dir_path)
+        return os.path.join(search_path, "*tomogram_metadata.json")
 
     @classmethod
     def _generate_hash_key(
@@ -34,6 +36,7 @@ def _generate_hash_key(
         return "-".join(
             [
                 container_key,
+                str(metadata.get("voxel_spacing", parents["voxel_spacing"].name)),
                 metadata.get("alignment_metadata_path", kwargs.get("alignment_metadata_path", "")),
                 metadata.get("reconstruction_method", ""),
                 metadata.get("processing", ""),

ingestion_tools/scripts/tests/s3_import/test_tomograms.py

@@ -4,7 +4,7 @@
 
 import pytest as pytest
 from importers.base_importer import BaseImporter
-from importers.tomogram import TomogramIdentifierHelper, TomogramImporter
+from importers.tomogram import TomogramImporter
 from importers.voxel_spacing import VoxelSpacingImporter
 from mypy_boto3_s3 import S3Client
 
@@ -82,13 +82,19 @@ def get_parents(config: DepositionImportConfig) -> dict[str, BaseImporter]:
             {"reconstruction_method": "SIRT", "processing": "raw"},
             "100-alignment_metadata.json",
             101,
-        ),
+        ),  # Existing metadata with same deposition_id but different reconstruction_method
         (
             10301,
             {"reconstruction_method": "WBP", "processing": "denoised"},
             "100-alignment_metadata.json",
             101,
-        ),
+        ),  # Existing metadata with same deposition_id but different processing
+        (
+            10301,
+            {"reconstruction_method": "WBP", "processing": "raw", "voxel_spacing": 20},
+            "100-alignment_metadata.json",
+            101,
+        ),  # Existing metadata with same deposition_id but different voxel_spacing
     ],
 )
 def test_tomogram_import(
@@ -106,15 +112,15 @@ def test_tomogram_import(
     parents = get_parents(config)
     run_name = parents["run"].name
     voxel_spacing = 13.48
-    prefix = f"output/{parents['dataset'].name}/{run_name}/Reconstructions/VoxelSpacing{voxel_spacing:.3f}/Tomograms"
+    prefix = f"output/{parents['dataset'].name}/{run_name}/Reconstructions/VoxelSpacing{{voxel_spacing:.3f}}/Tomograms"
     if deposition_id and existing_metadata:
         if alignment_path:
             existing_metadata["alignment_metadata_path"] = os.path.join(
                 test_output_bucket, f"output/{parents['dataset'].name}/{run_name}/Alignments", alignment_path,
             )
-        add_tomogram_metadata(prefix, deposition_id, existing_metadata)
-    TomogramIdentifierHelper.cached_identifiers.clear()
-    TomogramIdentifierHelper.loaded_containers.clear()
+        existing_prefix = prefix.format(voxel_spacing=existing_metadata.get("voxel_spacing", voxel_spacing))
+        add_tomogram_metadata(existing_prefix, deposition_id, existing_metadata)
+    prefix = prefix.format(voxel_spacing=voxel_spacing)
     tomogram = list(TomogramImporter.finder(config, **parents))
     for item in tomogram:
         item.import_item()