From 30b09b2c59656f9fefb726220211a2cccf899298 Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Tue, 9 Jul 2024 19:22:03 +0530 Subject: [PATCH 01/18] CSCKAN-290 migrations and data migrations --- ...alter_anatomicalentity_options_and_more.py | 119 ++++++++++++++++++ .../migrations/0053_auto_20240709_0909.py | 104 +++++++++++++++ .../migrations/0054_auto_20240708_1540.py | 31 +++++ 3 files changed, 254 insertions(+) create mode 100644 backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py create mode 100644 backend/composer/migrations/0053_auto_20240709_0909.py create mode 100644 backend/composer/migrations/0054_auto_20240708_1540.py diff --git a/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py b/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py new file mode 100644 index 00000000..02014275 --- /dev/null +++ b/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py @@ -0,0 +1,119 @@ +# Generated by Django 4.1.4 on 2024-07-08 13:18 + +from django.db import migrations, models +import django.db.models.deletion + + +def populate_intersection_anatomical_entity_meta_id_from_layer_region(apps, schema_editor): + """ + populating the new fields from AnatomicalEntityIntersection + layer_id -> layer_meta_id + region_id -> region_meta_id + """ + AnatomicalEntityIntersection = apps.get_model("composer", "AnatomicalEntityIntersection") + for intersection in AnatomicalEntityIntersection.objects.all(): + intersection.layer_meta_id = intersection.layer_id + intersection.region_meta_id = intersection.region_id + intersection.save() + + +def populate_layer_region_ae_meta_ids_from_anatomicalentitymeta_ptr_id(apps, schema_editor): + """ + populating the new fields from Layer and Region + anatomicalentitymeta_ptr_id -> layer_ae_meta_id + anatomicalentitymeta_ptr_id -> region_ae_meta_id + """ + Layer = apps.get_model("composer", "Layer") + Region = apps.get_model("composer", "Region") + + for layer in Layer.objects.all(): + layer.layer_ae_meta_id = layer.anatomicalentitymeta_ptr_id + layer.save() + + for region in Region.objects.all(): + region.region_ae_meta_id = region.anatomicalentitymeta_ptr_id + region.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("composer", "0051_alter_anatomicalentity_region_layer"), + ] + + # Steps taken in migration: + # 1. create new fields for layer and region - layer_ae_meta, region_ae_meta + # 2. create new fields for AnatomicalEntityIntersection - layer_meta, region_meta + # 3. populate the new fields in AnatomicalEntityIntersection from the old fields - layer, region + # 4. populate the new fields in Layer and Region from the old fields - anatomicalentitymeta_ptr_id + operations = [ + migrations.AlterModelOptions( + name="anatomicalentity", + options={ + "verbose_name": "Anatomical Entity", + "verbose_name_plural": "Anatomical Entities", + }, + ), + + migrations.AddField( + model_name="layer", + name="layer_ae_meta", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="layer_meta", + to="composer.anatomicalentitymeta", + ), + ), + migrations.AddField( + model_name="region", + name="region_ae_meta", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="region_meta", + to="composer.anatomicalentitymeta", + ), + ), + + # ------------------------ + migrations.AddField( + model_name="anatomicalentityintersection", + name="layer_meta", + field=models.ForeignKey( + default=1, + on_delete=django.db.models.deletion.CASCADE, + related_name="layer_intersection", + to="composer.anatomicalentitymeta", + ), + preserve_default=False, + ), + migrations.AddField( + model_name="anatomicalentityintersection", + name="region_meta", + field=models.ForeignKey( + default=1, + on_delete=django.db.models.deletion.CASCADE, + related_name="region_intersection", + to="composer.anatomicalentitymeta", + ), + preserve_default=False, + ), + + # ------------------------ + + migrations.RunPython( + code=populate_intersection_anatomical_entity_meta_id_from_layer_region, + reverse_code=migrations.RunPython.noop, + ), + + # ------------------------ + + migrations.RunPython( + code=populate_layer_region_ae_meta_ids_from_anatomicalentitymeta_ptr_id, + reverse_code=migrations.RunPython.noop, + ), + + ] + + diff --git a/backend/composer/migrations/0053_auto_20240709_0909.py b/backend/composer/migrations/0053_auto_20240709_0909.py new file mode 100644 index 00000000..a190b4c2 --- /dev/null +++ b/backend/composer/migrations/0053_auto_20240709_0909.py @@ -0,0 +1,104 @@ +# Generated by Django 4.1.4 on 2024-07-09 07:09 + +from django.db import migrations, models +import django.db.models.deletion + + + + + +class Migration(migrations.Migration): + + dependencies = [ + ("composer", "0052_alter_anatomicalentity_options_and_more"), + ] + + # Steps taken in migration: + # 1. create new models for layer and region - NewLayer, NewRegion + # 2. copy the necessary columns from the old models to the new models - layer_ae_meta, region_ae_meta + # 3. remove the old models - Layer, Region and it's fk from AnatomicalEntityIntersection + # 4. rename the new models to the old models - Layer, Region + + operations = [ + migrations.CreateModel( + name="NewLayer", + fields=[ + ( + "layer_id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + ), + ), + ( + "layer_ae_meta", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="layer_meta", + to="composer.anatomicalentitymeta", + ), + ) + ] + ), + migrations.RunSQL( + sql="INSERT INTO composer_newlayer (layer_ae_meta_id) SELECT anatomicalentitymeta_ptr_id FROM composer_layer", + reverse_sql="", + ), + migrations.RemoveField( + model_name="anatomicalentityintersection", + name="layer", + ), + migrations.DeleteModel( + name="Layer", + ), + migrations.RenameModel( + old_name="NewLayer", + new_name="Layer", + ), + + + # ---------------------------- + migrations.CreateModel( + name="NewRegion", + fields=[ + ( + "region_id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + ), + ), + ( + "region_ae_meta", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="region_meta", + to="composer.anatomicalentitymeta", + ), + ) + ] + ), + migrations.RunSQL( + sql="INSERT INTO composer_newregion (region_ae_meta_id) SELECT anatomicalentitymeta_ptr_id FROM composer_region", + reverse_sql="", + ), + migrations.RemoveField( + model_name="anatomicalentityintersection", + name="region", + ), + migrations.DeleteModel( + name="Region", + ), + migrations.RenameModel( + old_name="NewRegion", + new_name="Region", + ), + + + + + ] diff --git a/backend/composer/migrations/0054_auto_20240708_1540.py b/backend/composer/migrations/0054_auto_20240708_1540.py new file mode 100644 index 00000000..91e3011f --- /dev/null +++ b/backend/composer/migrations/0054_auto_20240708_1540.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.4 on 2024-07-08 13:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("composer", "0053_auto_20240709_0909"), + ] + + operations = [ + migrations.AlterModelOptions( + name="layer", + options={"verbose_name": "Layer", "verbose_name_plural": "Layers"}, + ), + migrations.AlterModelOptions( + name="region", + options={"verbose_name": "Region", "verbose_name_plural": "Regions"}, + ), + migrations.RenameField( + model_name="anatomicalentityintersection", + old_name="layer_meta", + new_name="layer", + ), + migrations.RenameField( + model_name="anatomicalentityintersection", + old_name="region_meta", + new_name="region", + ), + ] From e3ea2a89b098c9fa3418f0cce7abd94546e11f5d Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Tue, 9 Jul 2024 19:24:41 +0530 Subject: [PATCH 02/18] CSCKAN-290 new models and adjusting signal related --- backend/composer/models.py | 33 ++++++++++++++++++++++++++------- backend/composer/signals.py | 7 ++++--- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/backend/composer/models.py b/backend/composer/models.py index 9ab0a9d4..5a091ca0 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -240,18 +240,37 @@ class Meta: verbose_name_plural = "Anatomical Entities" -class Layer(AnatomicalEntityMeta): - ... +class Layer(models.Model): + layer_id = models.BigAutoField(primary_key=True, auto_created=True) + layer_ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_meta', null=True) + def __str__(self): + return self.layer_ae_meta.name + + class Meta: + verbose_name = "Layer" + verbose_name_plural = "Layers" -class Region(AnatomicalEntityMeta): - ... - layers = models.ManyToManyField(Layer, through='AnatomicalEntityIntersection') + + + + + +class Region(models.Model): + region_id = models.BigAutoField(primary_key=True, auto_created=True) + region_ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_meta', null=True) + + def __str__(self): + return self.region_ae_meta.name + + class Meta: + verbose_name = "Region" + verbose_name_plural = "Regions" class AnatomicalEntityIntersection(models.Model): - layer = models.ForeignKey(Layer, on_delete=models.CASCADE) - region = models.ForeignKey(Region, on_delete=models.CASCADE) + layer = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_intersection') + region = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_intersection') class Meta: verbose_name = "Region/Layer Combination" diff --git a/backend/composer/signals.py b/backend/composer/signals.py index a8690f8f..2cde3ab8 100644 --- a/backend/composer/signals.py +++ b/backend/composer/signals.py @@ -6,7 +6,7 @@ from .enums import CSState, NoteType from .models import ConnectivityStatement, ExportBatch, Note, Sentence, Synonym, \ - AnatomicalEntity, Layer, Region + AnatomicalEntity, Layer, Region, AnatomicalEntityMeta from .services.export_services import compute_metrics, ConnectivityStatementStateService @@ -51,9 +51,10 @@ def post_transition_cs(sender, instance, name, source, target, **kwargs): @receiver(post_save, sender=Layer) def create_layer_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: - AnatomicalEntity.objects.create(simple_entity=instance) + AnatomicalEntity.objects.create(simple_entity=instance.layer_ae_meta) @receiver(post_save, sender=Region) def create_region_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: - AnatomicalEntity.objects.create(simple_entity=instance) + AnatomicalEntity.objects.create(simple_entity=instance.region_ae_meta) + From e6262712314e70626b8a917f098f557acb8f3904 Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Tue, 9 Jul 2024 22:07:08 +0530 Subject: [PATCH 03/18] CSCKAN-290 additional validations and admin changes for new model --- backend/composer/admin.py | 70 ++++++++++++++++++++++++++++++++------ backend/composer/models.py | 30 ++++++++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/backend/composer/admin.py b/backend/composer/admin.py index 108e27b1..f167cbfa 100644 --- a/backend/composer/admin.py +++ b/backend/composer/admin.py @@ -98,11 +98,15 @@ class SentenceAdmin( ) class AnatomicalEntityAdmin(admin.ModelAdmin): - search_fields = ('simple_entity__name', 'region_layer__layer__name', 'region_layer__region__name') - autocomplete_fields = ('simple_entity', 'region_layer') - list_display = ('simple_entity', 'region_layer', "synonyms", "ontology_uri") + list_display = ('simple_entity', 'region_layer', 'synonyms', 'ontology_uri' ) list_display_links = ('simple_entity', 'region_layer') + search_fields = ( + 'simple_entity__name', 'simple_entity__ontology_uri', + 'region_layer__layer__name', 'region_layer__region__name', + 'region_layer__layer__ontology_uri', 'region_layer__region__ontology_uri', + ) inlines = (SynonymInline,) + autocomplete_fields = ('simple_entity', 'region_layer') # we need to make efficient queries to the database to get the list of anatomical entities def get_queryset(self, request: HttpRequest) -> QuerySet[Any]: @@ -130,22 +134,66 @@ def get_model_perms(self, request): return {} +class LayerAdminForm(forms.ModelForm): + class Meta: + model = Layer + fields = '__all__' + labels = { + 'layer_ae_meta': 'layer', + } + + +class RegionAdminForm(forms.ModelForm): + class Meta: + model = Region + fields = '__all__' + labels = { + 'region_ae_meta': 'region', + } + + class LayerAdmin(admin.ModelAdmin): - list_display = ('name', 'ontology_uri',) - search_fields = ('name',) + form = LayerAdminForm + list_display = ('layer_name', 'ontology_uri') + list_display_links = ('layer_name', 'ontology_uri') + search_fields = ('layer_ae_meta__name', 'layer_ae_meta__ontology_uri') + autocomplete_fields = ('layer_ae_meta',) + + @admin.display(description="Layer Name") + def layer_name(self, obj): + return obj.layer_ae_meta.name + + @admin.display(description="Ontology URI") + def ontology_uri(self, obj): + return obj.layer_ae_meta.ontology_uri + class RegionAdmin(admin.ModelAdmin): - list_display = ('name', 'ontology_uri',) - search_fields = ('name',) - filter_horizontal = ('layers',) + form = RegionAdminForm + list_display = ('region_name', 'ontology_uri') + list_display_links = ('region_name', 'ontology_uri') + search_fields = ('region_ae_meta__name', 'region_ae_meta__ontology_uri') + autocomplete_fields = ('region_ae_meta',) + + @admin.display(description="Region Name") + def region_name(self, obj): + return obj.region_ae_meta.name + + @admin.display(description="Ontology URI") + def ontology_uri(self, obj): + return obj.region_ae_meta.ontology_uri + class AnatomicalEntityIntersectionAdmin(nested_admin.NestedModelAdmin, admin.ModelAdmin): list_display = ('layer', 'region') - search_fields = ("region__name", "layer__name") - list_filter = ('layer', 'region',) - raw_id_fields = ('layer', 'region',) + raw_id_fields = ('layer', 'region') + list_filter = ('layer', 'region') + search_fields = ( + 'layer__layer_ae_meta__name', 'region__region_ae_meta__name', + 'layer__layer_ae_meta__ontology_uri', 'region__region_ae_meta__ontology_uri' + ) def get_model_perms(self, request): return {} diff --git a/backend/composer/models.py b/backend/composer/models.py index 5a091ca0..7636f191 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -4,6 +4,7 @@ from django.db.models.expressions import F from django.forms.widgets import Input as InputWidget from django_fsm import FSMField, transition +from django.core.exceptions import ValidationError from composer.services.state_services import ( ConnectivityStatementStateService, @@ -246,6 +247,21 @@ class Layer(models.Model): def __str__(self): return self.layer_ae_meta.name + + def clean(self): + if Layer.objects.filter(layer_ae_meta=self.layer_ae_meta).exists(): + raise ValidationError('Layer with this ontology_uri already exists') + if Region.objects.filter(region_ae_meta=self.layer_ae_meta).exists(): + raise ValidationError('Region with this ontology_uri already exists') + + + def save(self, *args, **kwargs): + if Layer.objects.filter(layer_ae_meta=self.layer_ae_meta).exists(): + raise ValueError('Layer with this Anatomical Entity Meta already exists') + elif Region.objects.filter(region_ae_meta=self.layer_ae_meta).exists(): + raise ValueError('Region with this Anatomical Entity Meta already exists') + else: + super().save(*args, **kwargs) class Meta: verbose_name = "Layer" @@ -263,6 +279,20 @@ class Region(models.Model): def __str__(self): return self.region_ae_meta.name + def clean(self): + if Layer.objects.filter(layer_ae_meta=self.region_ae_meta).exists(): + raise ValidationError('Layer with this ontology_uri already exists') + if Region.objects.filter(region_ae_meta=self.region_ae_meta).exists(): + raise ValidationError('Region with this ontology_uri already exists') + + def save(self, *args, **kwargs): + if Layer.objects.filter(layer_ae_meta=self.region_ae_meta).exists(): + raise ValidationError('Layer with this ontology_uri already exists') + elif Region.objects.filter(region_ae_meta=self.region_ae_meta).exists(): + raise ValidationError('Region with this ontology_uri already exists') + else: + super().save(*args, **kwargs) + class Meta: verbose_name = "Region" verbose_name_plural = "Regions" From 7614021bfe2ce6a02419f7d64ad5df6b5519c3fa Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Wed, 10 Jul 2024 00:08:20 +0530 Subject: [PATCH 04/18] CSCKAN-290 ingestion script update - for new models --- .../helpers/anatomical_entities_helper.py | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py index 81f5f3fa..8f6ba8b9 100644 --- a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py +++ b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py @@ -69,19 +69,19 @@ def add_entity_to_instance(instance, entity_field, entity, update_anatomic_entit def get_or_create_complex_entity(region_uri, layer_uri, update_anatomical_entities=False): try: - layer = Layer.objects.get(ontology_uri=layer_uri) + layer = Layer.objects.get(layer_ae_meta__ontology_uri=layer_uri) except Layer.DoesNotExist: layer = None try: - region = Region.objects.get(ontology_uri=region_uri) + region = Region.objects.get(region_ae_meta__ontology_uri=region_uri) except Region.DoesNotExist: region = None if update_anatomical_entities: if not layer: try: - layer_meta = AnatomicalEntityMeta.objects.get(ontology_uri=layer_uri) + layer_meta = AnatomicalEntityMeta.objects.get(ontology_uri=layer_uri) layer, _ = convert_anatomical_entity_to_specific_type(layer_meta, Layer) except AnatomicalEntityMeta.DoesNotExist: raise EntityNotFoundException(f"Layer meta not found for URI: {layer_uri}") @@ -96,7 +96,7 @@ def get_or_create_complex_entity(region_uri, layer_uri, update_anatomical_entiti if not layer or not region: raise EntityNotFoundException("Required Layer or Region not found.") - intersection, _ = AnatomicalEntityIntersection.objects.get_or_create(layer=layer, region=region) + intersection, _ = AnatomicalEntityIntersection.objects.get_or_create(layer=layer.layer_ae_meta, region=region.region_ae_meta) anatomical_entity, created = AnatomicalEntity.objects.get_or_create(region_layer=intersection) return anatomical_entity, created @@ -117,16 +117,23 @@ def convert_anatomical_entity_to_specific_type(entity_meta, target_model): Attempts to delete the original instance and create the new specific instance atomically. """ defaults = {'name': entity_meta.name, 'ontology_uri': entity_meta.ontology_uri} + try: + ae_instance = AnatomicalEntity.objects.get(simple_entity=entity_meta) + except AnatomicalEntity.DoesNotExist: + ae_instance = None try: with transaction.atomic(): - # Delete the anatomical entity meta in the incorrect type - entity_meta.delete() - # Create a new anatomical entity in the new specific type - specific_entity, created = target_model.objects.get_or_create( - ontology_uri=entity_meta.ontology_uri, - defaults=defaults - ) + # Delete the anatomical entity instance if it exists + if ae_instance: + ae_instance.delete() + + # Depending on the target_model, we need to create the specific entity - Layer/Region + specific_entity, created = None, False + if target_model == Layer: + specific_entity, created = Layer.objects.get_or_create(layer_ae_meta=entity_meta) + elif target_model == Region: + specific_entity, created = Region.objects.get_or_create(region_ae_meta=entity_meta) return specific_entity, created except IntegrityError as e: raise IntegrityError( From 194dbaff722961e40a05a73892dd937d17813286 Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Wed, 10 Jul 2024 00:32:48 +0530 Subject: [PATCH 05/18] CSCKAN-290 - rename layer_ae_meta and region_ae_meta to ae_meta --- backend/composer/admin.py | 24 +++++++++---------- ...me_layer_ae_meta_layer_ae_meta_and_more.py | 23 ++++++++++++++++++ backend/composer/models.py | 24 +++++++++---------- .../helpers/anatomical_entities_helper.py | 10 ++++---- backend/composer/signals.py | 4 ++-- 5 files changed, 54 insertions(+), 31 deletions(-) create mode 100644 backend/composer/migrations/0055_rename_layer_ae_meta_layer_ae_meta_and_more.py diff --git a/backend/composer/admin.py b/backend/composer/admin.py index f167cbfa..97b2b6f6 100644 --- a/backend/composer/admin.py +++ b/backend/composer/admin.py @@ -139,7 +139,7 @@ class Meta: model = Layer fields = '__all__' labels = { - 'layer_ae_meta': 'layer', + 'ae_meta': 'layer', } @@ -148,7 +148,7 @@ class Meta: model = Region fields = '__all__' labels = { - 'region_ae_meta': 'region', + 'ae_meta': 'region', } @@ -156,16 +156,16 @@ class LayerAdmin(admin.ModelAdmin): form = LayerAdminForm list_display = ('layer_name', 'ontology_uri') list_display_links = ('layer_name', 'ontology_uri') - search_fields = ('layer_ae_meta__name', 'layer_ae_meta__ontology_uri') - autocomplete_fields = ('layer_ae_meta',) + search_fields = ('ae_meta__name', 'ae_meta__ontology_uri') + autocomplete_fields = ('ae_meta',) @admin.display(description="Layer Name") def layer_name(self, obj): - return obj.layer_ae_meta.name + return obj.ae_meta.name @admin.display(description="Ontology URI") def ontology_uri(self, obj): - return obj.layer_ae_meta.ontology_uri + return obj.ae_meta.ontology_uri @@ -173,16 +173,16 @@ class RegionAdmin(admin.ModelAdmin): form = RegionAdminForm list_display = ('region_name', 'ontology_uri') list_display_links = ('region_name', 'ontology_uri') - search_fields = ('region_ae_meta__name', 'region_ae_meta__ontology_uri') - autocomplete_fields = ('region_ae_meta',) + search_fields = ('ae_meta__name', 'ae_meta__ontology_uri') + autocomplete_fields = ('ae_meta',) @admin.display(description="Region Name") def region_name(self, obj): - return obj.region_ae_meta.name + return obj.ae_meta.name @admin.display(description="Ontology URI") def ontology_uri(self, obj): - return obj.region_ae_meta.ontology_uri + return obj.ae_meta.ontology_uri @@ -191,8 +191,8 @@ class AnatomicalEntityIntersectionAdmin(nested_admin.NestedModelAdmin, admin.Mod raw_id_fields = ('layer', 'region') list_filter = ('layer', 'region') search_fields = ( - 'layer__layer_ae_meta__name', 'region__region_ae_meta__name', - 'layer__layer_ae_meta__ontology_uri', 'region__region_ae_meta__ontology_uri' + 'layer__ae_meta__name', 'region__ae_meta__name', + 'layer__ae_meta__ontology_uri', 'region__ae_meta__ontology_uri' ) def get_model_perms(self, request): diff --git a/backend/composer/migrations/0055_rename_layer_ae_meta_layer_ae_meta_and_more.py b/backend/composer/migrations/0055_rename_layer_ae_meta_layer_ae_meta_and_more.py new file mode 100644 index 00000000..05cbdc46 --- /dev/null +++ b/backend/composer/migrations/0055_rename_layer_ae_meta_layer_ae_meta_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.4 on 2024-07-09 19:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("composer", "0054_auto_20240708_1540"), + ] + + operations = [ + migrations.RenameField( + model_name="layer", + old_name="layer_ae_meta", + new_name="ae_meta", + ), + migrations.RenameField( + model_name="region", + old_name="region_ae_meta", + new_name="ae_meta", + ), + ] diff --git a/backend/composer/models.py b/backend/composer/models.py index 7636f191..3854621e 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -243,22 +243,22 @@ class Meta: class Layer(models.Model): layer_id = models.BigAutoField(primary_key=True, auto_created=True) - layer_ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_meta', null=True) + ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_meta', null=True) def __str__(self): - return self.layer_ae_meta.name + return self.ae_meta.name def clean(self): - if Layer.objects.filter(layer_ae_meta=self.layer_ae_meta).exists(): + if Layer.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Layer with this ontology_uri already exists') - if Region.objects.filter(region_ae_meta=self.layer_ae_meta).exists(): + if Region.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Region with this ontology_uri already exists') def save(self, *args, **kwargs): - if Layer.objects.filter(layer_ae_meta=self.layer_ae_meta).exists(): + if Layer.objects.filter(ae_meta=self.ae_meta).exists(): raise ValueError('Layer with this Anatomical Entity Meta already exists') - elif Region.objects.filter(region_ae_meta=self.layer_ae_meta).exists(): + elif Region.objects.filter(ae_meta=self.ae_meta).exists(): raise ValueError('Region with this Anatomical Entity Meta already exists') else: super().save(*args, **kwargs) @@ -274,21 +274,21 @@ class Meta: class Region(models.Model): region_id = models.BigAutoField(primary_key=True, auto_created=True) - region_ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_meta', null=True) + ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_meta', null=True) def __str__(self): - return self.region_ae_meta.name + return self.ae_meta.name def clean(self): - if Layer.objects.filter(layer_ae_meta=self.region_ae_meta).exists(): + if Layer.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Layer with this ontology_uri already exists') - if Region.objects.filter(region_ae_meta=self.region_ae_meta).exists(): + if Region.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Region with this ontology_uri already exists') def save(self, *args, **kwargs): - if Layer.objects.filter(layer_ae_meta=self.region_ae_meta).exists(): + if Layer.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Layer with this ontology_uri already exists') - elif Region.objects.filter(region_ae_meta=self.region_ae_meta).exists(): + elif Region.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Region with this ontology_uri already exists') else: super().save(*args, **kwargs) diff --git a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py index 8f6ba8b9..184c64f8 100644 --- a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py +++ b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py @@ -69,12 +69,12 @@ def add_entity_to_instance(instance, entity_field, entity, update_anatomic_entit def get_or_create_complex_entity(region_uri, layer_uri, update_anatomical_entities=False): try: - layer = Layer.objects.get(layer_ae_meta__ontology_uri=layer_uri) + layer = Layer.objects.get(ae_meta__ontology_uri=layer_uri) except Layer.DoesNotExist: layer = None try: - region = Region.objects.get(region_ae_meta__ontology_uri=region_uri) + region = Region.objects.get(ae_meta__ontology_uri=region_uri) except Region.DoesNotExist: region = None @@ -96,7 +96,7 @@ def get_or_create_complex_entity(region_uri, layer_uri, update_anatomical_entiti if not layer or not region: raise EntityNotFoundException("Required Layer or Region not found.") - intersection, _ = AnatomicalEntityIntersection.objects.get_or_create(layer=layer.layer_ae_meta, region=region.region_ae_meta) + intersection, _ = AnatomicalEntityIntersection.objects.get_or_create(layer=layer.ae_meta, region=region.ae_meta) anatomical_entity, created = AnatomicalEntity.objects.get_or_create(region_layer=intersection) return anatomical_entity, created @@ -131,9 +131,9 @@ def convert_anatomical_entity_to_specific_type(entity_meta, target_model): # Depending on the target_model, we need to create the specific entity - Layer/Region specific_entity, created = None, False if target_model == Layer: - specific_entity, created = Layer.objects.get_or_create(layer_ae_meta=entity_meta) + specific_entity, created = Layer.objects.get_or_create(ae_meta=entity_meta) elif target_model == Region: - specific_entity, created = Region.objects.get_or_create(region_ae_meta=entity_meta) + specific_entity, created = Region.objects.get_or_create(ae_meta=entity_meta) return specific_entity, created except IntegrityError as e: raise IntegrityError( diff --git a/backend/composer/signals.py b/backend/composer/signals.py index 2cde3ab8..05434088 100644 --- a/backend/composer/signals.py +++ b/backend/composer/signals.py @@ -51,10 +51,10 @@ def post_transition_cs(sender, instance, name, source, target, **kwargs): @receiver(post_save, sender=Layer) def create_layer_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: - AnatomicalEntity.objects.create(simple_entity=instance.layer_ae_meta) + AnatomicalEntity.objects.create(simple_entity=instance.ae_meta) @receiver(post_save, sender=Region) def create_region_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: - AnatomicalEntity.objects.create(simple_entity=instance.region_ae_meta) + AnatomicalEntity.objects.create(simple_entity=instance.ae_meta) From c58541928f6467c8697dea26e3c2048237a45674 Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Wed, 10 Jul 2024 00:33:36 +0530 Subject: [PATCH 06/18] CSCKAN-290 update the validator logic to ontology_uri --- backend/composer/services/cs_ingestion/helpers/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/composer/services/cs_ingestion/helpers/validators.py b/backend/composer/services/cs_ingestion/helpers/validators.py index 12312e96..2ab8c3da 100644 --- a/backend/composer/services/cs_ingestion/helpers/validators.py +++ b/backend/composer/services/cs_ingestion/helpers/validators.py @@ -92,5 +92,5 @@ def annotate_invalid_forward_connections(statement: Dict, statement_ids: Set[str def found_entity(uri: str, Model: Type[DjangoModel] = None) -> bool: if Model: - return Model.objects.filter(ontology_uri=uri).exists() + return Model.objects.filter(ae_meta__ontology_uri=uri).exists() return AnatomicalEntityMeta.objects.filter(ontology_uri=uri).exists() From 5931c6ba128b1fcefff04b8461d58b05138245bb Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Wed, 10 Jul 2024 02:50:27 +0530 Subject: [PATCH 07/18] CSCKAN-290 update helpers annd signals - get_or_create instead of create --- .../cs_ingestion/helpers/anatomical_entities_helper.py | 8 -------- backend/composer/signals.py | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py index 184c64f8..5ef83b28 100644 --- a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py +++ b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py @@ -117,17 +117,9 @@ def convert_anatomical_entity_to_specific_type(entity_meta, target_model): Attempts to delete the original instance and create the new specific instance atomically. """ defaults = {'name': entity_meta.name, 'ontology_uri': entity_meta.ontology_uri} - try: - ae_instance = AnatomicalEntity.objects.get(simple_entity=entity_meta) - except AnatomicalEntity.DoesNotExist: - ae_instance = None try: with transaction.atomic(): - # Delete the anatomical entity instance if it exists - if ae_instance: - ae_instance.delete() - # Depending on the target_model, we need to create the specific entity - Layer/Region specific_entity, created = None, False if target_model == Layer: diff --git a/backend/composer/signals.py b/backend/composer/signals.py index 05434088..3f9f25d5 100644 --- a/backend/composer/signals.py +++ b/backend/composer/signals.py @@ -51,10 +51,10 @@ def post_transition_cs(sender, instance, name, source, target, **kwargs): @receiver(post_save, sender=Layer) def create_layer_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: - AnatomicalEntity.objects.create(simple_entity=instance.ae_meta) + AnatomicalEntity.objects.get_or_create(simple_entity=instance.ae_meta) @receiver(post_save, sender=Region) def create_region_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: - AnatomicalEntity.objects.create(simple_entity=instance.ae_meta) + AnatomicalEntity.objects.get_or_create(simple_entity=instance.ae_meta) From b53d664a9e187875c7dc3d5126a109ebe7dc472f Mon Sep 17 00:00:00 2001 From: afonso Date: Wed, 10 Jul 2024 15:45:08 +0100 Subject: [PATCH 08/18] CSCKAN-290 fix: Update region and layer saving constraints --- backend/composer/models.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/backend/composer/models.py b/backend/composer/models.py index 3854621e..d783005e 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -251,17 +251,11 @@ def __str__(self): def clean(self): if Layer.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Layer with this ontology_uri already exists') - if Region.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValidationError('Region with this ontology_uri already exists') def save(self, *args, **kwargs): - if Layer.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValueError('Layer with this Anatomical Entity Meta already exists') - elif Region.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValueError('Region with this Anatomical Entity Meta already exists') - else: - super().save(*args, **kwargs) + self.clean() + super().save(*args, **kwargs) class Meta: verbose_name = "Layer" @@ -280,18 +274,12 @@ def __str__(self): return self.ae_meta.name def clean(self): - if Layer.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValidationError('Layer with this ontology_uri already exists') if Region.objects.filter(ae_meta=self.ae_meta).exists(): raise ValidationError('Region with this ontology_uri already exists') def save(self, *args, **kwargs): - if Layer.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValidationError('Layer with this ontology_uri already exists') - elif Region.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValidationError('Region with this ontology_uri already exists') - else: - super().save(*args, **kwargs) + self.clean() + super().save(*args, **kwargs) class Meta: verbose_name = "Region" From 79c8e36cfcddcc035474d2b572953b2d8c2ee8f1 Mon Sep 17 00:00:00 2001 From: afonso Date: Wed, 10 Jul 2024 15:47:22 +0100 Subject: [PATCH 09/18] CSCKAN-290 feat: Make anatomical entity meta relation mandatory for Region and Layer --- ...lter_layer_ae_meta_alter_region_ae_meta.py | 31 +++++++++++++++++++ backend/composer/models.py | 4 +-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 backend/composer/migrations/0056_alter_layer_ae_meta_alter_region_ae_meta.py diff --git a/backend/composer/migrations/0056_alter_layer_ae_meta_alter_region_ae_meta.py b/backend/composer/migrations/0056_alter_layer_ae_meta_alter_region_ae_meta.py new file mode 100644 index 00000000..04417506 --- /dev/null +++ b/backend/composer/migrations/0056_alter_layer_ae_meta_alter_region_ae_meta.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.4 on 2024-07-10 14:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0055_rename_layer_ae_meta_layer_ae_meta_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="layer", + name="ae_meta", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="layer_meta", + to="composer.anatomicalentitymeta", + ), + ), + migrations.AlterField( + model_name="region", + name="ae_meta", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="region_meta", + to="composer.anatomicalentitymeta", + ), + ), + ] diff --git a/backend/composer/models.py b/backend/composer/models.py index d783005e..4d70dc4d 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -243,7 +243,7 @@ class Meta: class Layer(models.Model): layer_id = models.BigAutoField(primary_key=True, auto_created=True) - ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_meta', null=True) + ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_meta', null=False) def __str__(self): return self.ae_meta.name @@ -268,7 +268,7 @@ class Meta: class Region(models.Model): region_id = models.BigAutoField(primary_key=True, auto_created=True) - ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_meta', null=True) + ae_meta = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_meta', null=False) def __str__(self): return self.ae_meta.name From cadc689b0b368bcf8f9af1afea4ba8fd48a26ba0 Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Wed, 10 Jul 2024 23:48:11 +0530 Subject: [PATCH 10/18] CSCKAN-290 update migration file for layer region in AEIntersection --- ...alter_anatomicalentity_options_and_more.py | 6 ++-- ...vitystatement_projection_valid_and_more.py | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 backend/composer/migrations/0057_remove_connectivitystatement_projection_valid_and_more.py diff --git a/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py b/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py index 02014275..df67c214 100644 --- a/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py +++ b/backend/composer/migrations/0052_alter_anatomicalentity_options_and_more.py @@ -81,23 +81,21 @@ class Migration(migrations.Migration): model_name="anatomicalentityintersection", name="layer_meta", field=models.ForeignKey( - default=1, + null=True, on_delete=django.db.models.deletion.CASCADE, related_name="layer_intersection", to="composer.anatomicalentitymeta", ), - preserve_default=False, ), migrations.AddField( model_name="anatomicalentityintersection", name="region_meta", field=models.ForeignKey( - default=1, + null=True, on_delete=django.db.models.deletion.CASCADE, related_name="region_intersection", to="composer.anatomicalentitymeta", ), - preserve_default=False, ), # ------------------------ diff --git a/backend/composer/migrations/0057_remove_connectivitystatement_projection_valid_and_more.py b/backend/composer/migrations/0057_remove_connectivitystatement_projection_valid_and_more.py new file mode 100644 index 00000000..3f7b9c3a --- /dev/null +++ b/backend/composer/migrations/0057_remove_connectivitystatement_projection_valid_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 4.1.4 on 2024-07-10 18:12 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("composer", "0056_alter_layer_ae_meta_alter_region_ae_meta"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="connectivitystatement", + name="projection_valid", + ), + migrations.AlterField( + model_name="anatomicalentityintersection", + name="layer", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="layer_intersection", + to="composer.anatomicalentitymeta", + ), + ), + migrations.AlterField( + model_name="anatomicalentityintersection", + name="region", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="region_intersection", + to="composer.anatomicalentitymeta", + ), + ), + ] From 922dc7fadd039ea1369585929320adbb90fc0c72 Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Wed, 10 Jul 2024 23:53:54 +0530 Subject: [PATCH 11/18] CSCKAN-293 - rename ae helper method - create instead of convert --- .../cs_ingestion/helpers/anatomical_entities_helper.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py index 5ef83b28..4cfe9849 100644 --- a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py +++ b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py @@ -82,14 +82,14 @@ def get_or_create_complex_entity(region_uri, layer_uri, update_anatomical_entiti if not layer: try: layer_meta = AnatomicalEntityMeta.objects.get(ontology_uri=layer_uri) - layer, _ = convert_anatomical_entity_to_specific_type(layer_meta, Layer) + layer, _ = create_region_layer_from_anatomical_entity(layer_meta, Layer) except AnatomicalEntityMeta.DoesNotExist: raise EntityNotFoundException(f"Layer meta not found for URI: {layer_uri}") if not region: try: region_meta = AnatomicalEntityMeta.objects.get(ontology_uri=region_uri) - region, _ = convert_anatomical_entity_to_specific_type(region_meta, Region) + region, _ = create_region_layer_from_anatomical_entity(region_meta, Region) except AnatomicalEntityMeta.DoesNotExist: raise EntityNotFoundException(f"Region meta not found for URI: {layer_uri}") else: @@ -111,10 +111,9 @@ def get_or_create_simple_entity(ontology_uri: str): raise EntityNotFoundException(f"Anatomical entity meta not found for URI: {ontology_uri}") -def convert_anatomical_entity_to_specific_type(entity_meta, target_model): +def create_region_layer_from_anatomical_entity(entity_meta, target_model): """ - Convert an AnatomicalEntityMeta instance to a more specific subclass type (Layer or Region). - Attempts to delete the original instance and create the new specific instance atomically. + Create a new specific entity (Layer or Region) from an AnatomicalEntityMeta instance. """ defaults = {'name': entity_meta.name, 'ontology_uri': entity_meta.ontology_uri} From e4cde9d750659200dda882b7209459aee07ce31f Mon Sep 17 00:00:00 2001 From: "D. Gopal Krishna" Date: Thu, 11 Jul 2024 20:16:22 +0530 Subject: [PATCH 12/18] CSCKAN-290 comment the liveness and readiness from composer.yaml --- k8s/composer.yaml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/k8s/composer.yaml b/k8s/composer.yaml index ee2e10f6..d785b835 100644 --- a/k8s/composer.yaml +++ b/k8s/composer.yaml @@ -35,24 +35,24 @@ spec: value: "{{DEBUG}}" ports: - containerPort: 8000 - livenessProbe: - failureThreshold: 3 - httpGet: - path: /admin/login/ - port: 8000 - scheme: HTTP - initialDelaySeconds: 45 - periodSeconds: 30 - timeoutSeconds: 2 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /admin/login/ - port: 8000 - scheme: HTTP - initialDelaySeconds: 15 - periodSeconds: 30 - timeoutSeconds: 2 + # livenessProbe: + # failureThreshold: 3 + # httpGet: + # path: /admin/login/ + # port: 8000 + # scheme: HTTP + # initialDelaySeconds: 45 + # periodSeconds: 30 + # timeoutSeconds: 2 + # readinessProbe: + # failureThreshold: 3 + # httpGet: + # path: /admin/login/ + # port: 8000 + # scheme: HTTP + # initialDelaySeconds: 15 + # periodSeconds: 30 + # timeoutSeconds: 2 resources: limits: cpu: 1500m From 0636030f75bab0d4ab302ee60d3e11deea97359a Mon Sep 17 00:00:00 2001 From: afonso Date: Mon, 15 Jul 2024 14:18:18 +0100 Subject: [PATCH 13/18] CSCKAN-290 fix: Update region layer serializers --- backend/composer/api/serializers.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/backend/composer/api/serializers.py b/backend/composer/api/serializers.py index 7259060e..405c090a 100644 --- a/backend/composer/api/serializers.py +++ b/backend/composer/api/serializers.py @@ -81,29 +81,6 @@ class Meta: dept = 2 -class LayerSerializer(serializers.ModelSerializer): - class Meta: - model = Layer - fields = ( - "id", - "name", - "ontology_uri", - ) - - -class RegionSerializer(serializers.ModelSerializer): - layers = LayerSerializer(many=True, read_only=True) - - class Meta: - model = Region - fields = ( - "id", - "name", - "ontology_uri", - "layers", - ) - - class AnatomicalEntityMetaSerializer(serializers.ModelSerializer): class Meta: model = AnatomicalEntityMeta @@ -115,8 +92,8 @@ class Meta: class AnatomicalEntityIntersectionSerializer(serializers.ModelSerializer): - layer = LayerSerializer(read_only=True) - region = RegionSerializer(read_only=True) + layer = AnatomicalEntityMetaSerializer(read_only=True) + region = AnatomicalEntityMetaSerializer(read_only=True) class Meta: model = AnatomicalEntityIntersection From 90a72319ea4531a36dbe046c3f89aa1732e3971f Mon Sep 17 00:00:00 2001 From: afonso Date: Wed, 17 Jul 2024 15:42:14 +0100 Subject: [PATCH 14/18] CSCKAN-290 feat: Add migration to remove duplicates in layers, regions or anatomical entity intersections --- .../migrations/0058_auto_20240717_1550.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 backend/composer/migrations/0058_auto_20240717_1550.py diff --git a/backend/composer/migrations/0058_auto_20240717_1550.py b/backend/composer/migrations/0058_auto_20240717_1550.py new file mode 100644 index 00000000..60d9cc46 --- /dev/null +++ b/backend/composer/migrations/0058_auto_20240717_1550.py @@ -0,0 +1,67 @@ +# Generated by Django 4.1.4 on 2024-07-17 13:50 + +from django.db import migrations +from django.db.models import Count + + +def remove_duplicate_layers(apps, schema_editor): + Layer = apps.get_model('composer', 'Layer') + duplicates = ( + Layer.objects + .values('ae_meta_id') + .annotate(count_id=Count('layer_id')) + .filter(count_id__gt=1) + ) + + for duplicate in duplicates: + layers = Layer.objects.filter(ae_meta_id=duplicate['ae_meta_id']).order_by('layer_id') + layers_to_delete = layers[1:] # Keep the first occurrence, delete the rest + for layer in layers_to_delete: + layer.delete() + + +def remove_duplicate_regions(apps, schema_editor): + Region = apps.get_model('composer', 'Region') + duplicates = ( + Region.objects + .values('ae_meta_id') + .annotate(count_id=Count('region_id')) + .filter(count_id__gt=1) + ) + + for duplicate in duplicates: + regions = Region.objects.filter(ae_meta_id=duplicate['ae_meta_id']).order_by('region_id') + regions_to_delete = regions[1:] # Keep the first occurrence, delete the rest + for region in regions_to_delete: + region.delete() + + +def remove_duplicate_intersections(apps, schema_editor): + AnatomicalEntityIntersection = apps.get_model('composer', 'AnatomicalEntityIntersection') + duplicates = ( + AnatomicalEntityIntersection.objects + .values('layer_id', 'region_id') + .annotate(count_id=Count('id')) + .filter(count_id__gt=1) + ) + + for duplicate in duplicates: + intersections = AnatomicalEntityIntersection.objects.filter( + layer_id=duplicate['layer_id'], + region_id=duplicate['region_id'] + ).order_by('id') + intersections_to_delete = intersections[1:] # Keep the first occurrence, delete the rest + for intersection in intersections_to_delete: + intersection.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0057_remove_connectivitystatement_projection_valid_and_more"), + ] + + operations = [ + migrations.RunPython(remove_duplicate_layers), + migrations.RunPython(remove_duplicate_regions), + migrations.RunPython(remove_duplicate_intersections), + ] From d9ee3fc1615673aecadee9c529099ce1cff28dc8 Mon Sep 17 00:00:00 2001 From: afonso Date: Wed, 17 Jul 2024 15:42:43 +0100 Subject: [PATCH 15/18] CSCKAN-290 feat: Add constraints to not allow duplicates in layers, regions, anatomical entity intersections --- ...nique_layer_region_combination_and_more.py | 30 ++++++++++++++++ backend/composer/models.py | 35 +++++++++---------- 2 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 backend/composer/migrations/0059_anatomicalentityintersection_unique_layer_region_combination_and_more.py diff --git a/backend/composer/migrations/0059_anatomicalentityintersection_unique_layer_region_combination_and_more.py b/backend/composer/migrations/0059_anatomicalentityintersection_unique_layer_region_combination_and_more.py new file mode 100644 index 00000000..3ab4c327 --- /dev/null +++ b/backend/composer/migrations/0059_anatomicalentityintersection_unique_layer_region_combination_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 4.1.4 on 2024-07-17 14:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0058_auto_20240717_1550"), + ] + + operations = [ + migrations.AddConstraint( + model_name="anatomicalentityintersection", + constraint=models.UniqueConstraint( + fields=("layer", "region"), name="unique_layer_region_combination" + ), + ), + migrations.AddConstraint( + model_name="layer", + constraint=models.UniqueConstraint( + fields=("ae_meta",), name="unique_layer_ae_meta" + ), + ), + migrations.AddConstraint( + model_name="region", + constraint=models.UniqueConstraint( + fields=("ae_meta",), name="unique_region_ae_meta" + ), + ), + ] diff --git a/backend/composer/models.py b/backend/composer/models.py index 78413358..df7bd743 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -98,7 +98,7 @@ def get_queryset(self): def excluding_draft(self): return self.get_queryset().exclude(state=CSState.DRAFT) - + def exported(self): return self.get_queryset().filter(state=CSState.EXPORTED) @@ -247,23 +247,16 @@ class Layer(models.Model): def __str__(self): return self.ae_meta.name - - def clean(self): - if Layer.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValidationError('Layer with this ontology_uri already exists') - def save(self, *args, **kwargs): - self.clean() super().save(*args, **kwargs) - + class Meta: verbose_name = "Layer" verbose_name_plural = "Layers" - - - - + constraints = [ + models.UniqueConstraint(fields=['ae_meta'], name='unique_layer_ae_meta') + ] class Region(models.Model): @@ -272,27 +265,31 @@ class Region(models.Model): def __str__(self): return self.ae_meta.name - - def clean(self): - if Region.objects.filter(ae_meta=self.ae_meta).exists(): - raise ValidationError('Region with this ontology_uri already exists') def save(self, *args, **kwargs): - self.clean() super().save(*args, **kwargs) - + class Meta: verbose_name = "Region" verbose_name_plural = "Regions" + constraints = [ + models.UniqueConstraint(fields=['ae_meta'], name='unique_region_ae_meta') + ] class AnatomicalEntityIntersection(models.Model): layer = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='layer_intersection') region = models.ForeignKey(AnatomicalEntityMeta, on_delete=models.CASCADE, related_name='region_intersection') + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + class Meta: verbose_name = "Region/Layer Combination" verbose_name_plural = "Region/Layer Combinations" + constraints = [ + models.UniqueConstraint(fields=['layer', 'region'], name='unique_layer_region_combination') + ] def __str__(self): return f"{self.region.name} ({self.layer.name})" @@ -309,7 +306,7 @@ def name(self): if self.region_layer: return str(self.region_layer) return 'Unknown Anatomical Entity' - + @property def ontology_uri(self): if self.simple_entity: From c9293b4f7f7224f5fadb8232bbdc31466c483492 Mon Sep 17 00:00:00 2001 From: afonso Date: Wed, 17 Jul 2024 15:43:40 +0100 Subject: [PATCH 16/18] CSCKAN-290 feat: Add on anatomical entity delete signal to delete associated_entities (simple or complex) --- backend/composer/signals.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/backend/composer/signals.py b/backend/composer/signals.py index 3f9f25d5..f0a40a8e 100644 --- a/backend/composer/signals.py +++ b/backend/composer/signals.py @@ -1,6 +1,6 @@ from django.core.exceptions import ValidationError from django.dispatch import receiver -from django.db.models.signals import post_save, m2m_changed, pre_save +from django.db.models.signals import post_save, m2m_changed, pre_save, post_delete from django_fsm.signals import post_transition @@ -48,13 +48,25 @@ def post_transition_cs(sender, instance, name, source, target, **kwargs): # add important tag to CS when transition to COMPOSE_NOW from NPO Approved or Exported instance = ConnectivityStatementStateService.add_important_tag(instance) + @receiver(post_save, sender=Layer) def create_layer_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: AnatomicalEntity.objects.get_or_create(simple_entity=instance.ae_meta) + @receiver(post_save, sender=Region) def create_region_anatomical_entity(sender, instance=None, created=False, **kwargs): if created and instance: AnatomicalEntity.objects.get_or_create(simple_entity=instance.ae_meta) + +@receiver(post_delete, sender=AnatomicalEntity) +def delete_associated_entities(sender, instance, **kwargs): + # Delete the associated simple_entity if it exists + if instance.simple_entity: + instance.simple_entity.delete() + + # Delete the associated region_layer if it exists + if instance.region_layer: + instance.region_layer.delete() \ No newline at end of file From 275729603e0574b1893f693720352fa5c3725f39 Mon Sep 17 00:00:00 2001 From: afonso Date: Thu, 18 Jul 2024 11:12:48 +0100 Subject: [PATCH 17/18] CSCKAN-290 fix: Make export more robust --- backend/composer/services/export_services.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/composer/services/export_services.py b/backend/composer/services/export_services.py index 90e27895..d80bbe22 100644 --- a/backend/composer/services/export_services.py +++ b/backend/composer/services/export_services.py @@ -336,9 +336,11 @@ def get_soma_phenotype_row(cs: ConnectivityStatement): def get_phenotype_row(cs: ConnectivityStatement): + phenotype_name = cs.phenotype.name if cs.phenotype else "" + phenotype_ontology_uri = cs.phenotype.ontology_uri if cs.phenotype else "" return Row( - cs.phenotype.name, - cs.phenotype.ontology_uri, + phenotype_name, + phenotype_ontology_uri, ExportRelationships.hasAnatomicalSystemPhenotype.label, ExportRelationships.hasAnatomicalSystemPhenotype.value, "", @@ -347,9 +349,12 @@ def get_phenotype_row(cs: ConnectivityStatement): def get_projection_phenotype_row(cs: ConnectivityStatement): + projection_phenotype = cs.projection_phenotype if cs.projection_phenotype else "" + projection_phenotype_ontology_uri = cs.projection_phenotype.ontology_uri if cs.projection_phenotype else "" + return Row( - cs.projection_phenotype, - cs.projection_phenotype.ontology_uri, + projection_phenotype, + projection_phenotype_ontology_uri, ExportRelationships.hasProjectionPhenotype.label, ExportRelationships.hasProjectionPhenotype.value, "", From b7352be1758185953106b225838a1372c93a8149 Mon Sep 17 00:00:00 2001 From: afonso Date: Thu, 18 Jul 2024 13:23:04 +0100 Subject: [PATCH 18/18] CSCKAN-290 fix: Make export more robust --- backend/composer/services/export_services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/composer/services/export_services.py b/backend/composer/services/export_services.py index d80bbe22..2ab27127 100644 --- a/backend/composer/services/export_services.py +++ b/backend/composer/services/export_services.py @@ -100,7 +100,8 @@ def get_neuron_population_label(cs: ConnectivityStatement, row: Row): def get_type(cs: ConnectivityStatement, row: Row): - return cs.phenotype.name + return cs.phenotype.name if cs.phenotype else "" + def get_structure(cs: ConnectivityStatement, row: Row):