From 9549f422528cc4bbd813873a7f9783a584d42a92 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 8 Mar 2023 10:57:16 +0300 Subject: [PATCH 1/6] Add indexes to instance model date fields Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/instance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 3bc59b1221..709530bb3a 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -652,19 +652,19 @@ class Instance(models.Model, InstanceBaseClass): survey_type = models.ForeignKey("logger.SurveyType", on_delete=models.PROTECT) # shows when we first received this instance - date_created = models.DateTimeField(auto_now_add=True) + date_created = models.DateTimeField(auto_now_add=True, db_index=True) # this will end up representing "date last parsed" - date_modified = models.DateTimeField(auto_now=True) + date_modified = models.DateTimeField(auto_now=True, db_index=True) # this will end up representing "date instance was deleted" - deleted_at = models.DateTimeField(null=True, default=None) + deleted_at = models.DateTimeField(null=True, default=None, db_index=True) deleted_by = models.ForeignKey( User, related_name="deleted_instances", null=True, on_delete=models.SET_NULL ) # this will be edited when we need to create a new InstanceHistory object - last_edited = models.DateTimeField(null=True, default=None) + last_edited = models.DateTimeField(null=True, default=None, db_index=True) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete From fe4c26f276d6b434101f80fc67bd042c978dfcad Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 8 Mar 2023 10:58:32 +0300 Subject: [PATCH 2/6] Add migration for adding indexes to date fields on the instance model Signed-off-by: Kipchirchir Sigei --- .../migrations/0005_auto_20230308_0252.py | 23 +++++++++++++++++++ onadata/apps/logger/models/instance.py | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 onadata/apps/logger/migrations/0005_auto_20230308_0252.py diff --git a/onadata/apps/logger/migrations/0005_auto_20230308_0252.py b/onadata/apps/logger/migrations/0005_auto_20230308_0252.py new file mode 100644 index 0000000000..23e17c9524 --- /dev/null +++ b/onadata/apps/logger/migrations/0005_auto_20230308_0252.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.16 on 2023-03-08 07:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0004_update_instance_geoms'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='date_created', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='instance', + name='date_modified', + field=models.DateTimeField(auto_now=True, db_index=True), + ), + ] diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 709530bb3a..942e0c25f3 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -658,13 +658,13 @@ class Instance(models.Model, InstanceBaseClass): date_modified = models.DateTimeField(auto_now=True, db_index=True) # this will end up representing "date instance was deleted" - deleted_at = models.DateTimeField(null=True, default=None, db_index=True) + deleted_at = models.DateTimeField(null=True, default=None) deleted_by = models.ForeignKey( User, related_name="deleted_instances", null=True, on_delete=models.SET_NULL ) # this will be edited when we need to create a new InstanceHistory object - last_edited = models.DateTimeField(null=True, default=None, db_index=True) + last_edited = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete From d8d1febd7e46b4e770fac9b23f787a42deda3a76 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 9 Mar 2023 10:07:49 +0300 Subject: [PATCH 3/6] Apply indexes concurrently to prevent lock writes Signed-off-by: Kipchirchir Sigei --- .../migrations/0004_update_instance_geoms.py | 13 +++-- .../migrations/0005_auto_20230308_0252.py | 47 ++++++++++++++----- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/onadata/apps/logger/migrations/0004_update_instance_geoms.py b/onadata/apps/logger/migrations/0004_update_instance_geoms.py index 59fb98c4db..6ef70c3fe6 100644 --- a/onadata/apps/logger/migrations/0004_update_instance_geoms.py +++ b/onadata/apps/logger/migrations/0004_update_instance_geoms.py @@ -2,16 +2,19 @@ from django.db import migrations from onadata.apps.logger.models.instance import Instance +from onadata.libs.utils.model_tools import queryset_iterator def update_instance_geoms(apps, schema_editor): """ Update instance geom field with valid geom values """ - for inst in Instance.objects.filter( - deleted_at__isnull=True, - xform__downloadable=True, - xform__deleted_at__isnull=True, + for inst in queryset_iterator( + Instance.objects.filter( + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True, + ) ): if inst.geom and inst.geom.empty: inst.geom = None @@ -21,7 +24,7 @@ def update_instance_geoms(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0003_alter_instance_media_all_received'), + ("logger", "0003_alter_instance_media_all_received"), ] operations = [migrations.RunPython(update_instance_geoms)] diff --git a/onadata/apps/logger/migrations/0005_auto_20230308_0252.py b/onadata/apps/logger/migrations/0005_auto_20230308_0252.py index 23e17c9524..485a8bfa6f 100644 --- a/onadata/apps/logger/migrations/0005_auto_20230308_0252.py +++ b/onadata/apps/logger/migrations/0005_auto_20230308_0252.py @@ -4,20 +4,45 @@ class Migration(migrations.Migration): + atomic = False dependencies = [ - ('logger', '0004_update_instance_geoms'), + ("logger", "0004_update_instance_geoms"), ] operations = [ - migrations.AlterField( - model_name='instance', - name='date_created', - field=models.DateTimeField(auto_now_add=True, db_index=True), - ), - migrations.AlterField( - model_name='instance', - name='date_modified', - field=models.DateTimeField(auto_now=True, db_index=True), - ), + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql=""" + CREATE INDEX "logger_instance_date_created_b2427770" + ON "logger_instance" ("date_created"); + """, + reverse_sql=""" + DROP INDEX "logger_instance_date_created_b2427770"; + """, + ), + migrations.RunSQL( + sql=""" + CREATE INDEX "logger_instance_date_modified_a32599fc" + ON "logger_instance" ("date_modified"); + """, + reverse_sql=""" + DROP INDEX "logger_instance_date_modified_a32599fc"; + """, + ), + ], + state_operations=[ + migrations.AlterField( + model_name="instance", + name="date_created", + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name="instance", + name="date_modified", + field=models.DateTimeField(auto_now=True, db_index=True), + ), + ], + ) ] From fa6f2def3dc5ff11d7cc2e7a64acd08fbcb72b79 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Tue, 14 Mar 2023 16:47:37 +0300 Subject: [PATCH 4/6] cleanup Signed-off-by: Kipchirchir Sigei --- .../0005_add_date_fields_indexing.py | 54 +++++++++++++++++++ .../migrations/0005_auto_20230308_0252.py | 48 ----------------- onadata/apps/logger/models/instance.py | 9 +++- 3 files changed, 61 insertions(+), 50 deletions(-) create mode 100644 onadata/apps/logger/migrations/0005_add_date_fields_indexing.py delete mode 100644 onadata/apps/logger/migrations/0005_auto_20230308_0252.py diff --git a/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py b/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py new file mode 100644 index 0000000000..f5af866ac6 --- /dev/null +++ b/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py @@ -0,0 +1,54 @@ +# Generated by Django 3.2.18 on 2023-03-14 12:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('logger', '0004_update_instance_geoms'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql='CREATE INDEX CONCURRENTLY "logger_inst_date_cr_42899d_idx" ON "logger_instance" ("date_created");', + reverse_sql='DROP INDEX "logger_inst_date_cr_42899d_idx";', + ), + migrations.RunSQL( + sql='CREATE INDEX CONCURRENTLY "logger_inst_date_mo_5a1bd3_idx" ON "logger_instance" ("date_modified");', + reverse_sql='DROP INDEX "logger_inst_date_mo_5a1bd3_idx";', + ), + migrations.RunSQL( + sql='CREATE INDEX CONCURRENTLY "logger_inst_deleted_da31a3_idx" ON "logger_instance" ("deleted_at");', + reverse_sql='DROP INDEX "logger_inst_deleted_da31a3_idx";', + ), + ], + state_operations=[ + migrations.AddIndex( + model_name="instance", + index=models.Index( + fields=["date_created"], name="logger_inst_date_cr_42899d_idx" + ), + ), + migrations.AddIndex( + model_name="instance", + index=models.Index( + fields=["date_modified"], name="logger_inst_date_mo_5a1bd3_idx" + ), + ), + migrations.AddIndex( + model_name="instance", + index=models.Index( + fields=["deleted_at"], name="logger_inst_deleted_da31a3_idx" + ), + ), + ], + ) + ] + + + + diff --git a/onadata/apps/logger/migrations/0005_auto_20230308_0252.py b/onadata/apps/logger/migrations/0005_auto_20230308_0252.py deleted file mode 100644 index 485a8bfa6f..0000000000 --- a/onadata/apps/logger/migrations/0005_auto_20230308_0252.py +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by Django 3.2.16 on 2023-03-08 07:52 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - atomic = False - - dependencies = [ - ("logger", "0004_update_instance_geoms"), - ] - - operations = [ - migrations.SeparateDatabaseAndState( - database_operations=[ - migrations.RunSQL( - sql=""" - CREATE INDEX "logger_instance_date_created_b2427770" - ON "logger_instance" ("date_created"); - """, - reverse_sql=""" - DROP INDEX "logger_instance_date_created_b2427770"; - """, - ), - migrations.RunSQL( - sql=""" - CREATE INDEX "logger_instance_date_modified_a32599fc" - ON "logger_instance" ("date_modified"); - """, - reverse_sql=""" - DROP INDEX "logger_instance_date_modified_a32599fc"; - """, - ), - ], - state_operations=[ - migrations.AlterField( - model_name="instance", - name="date_created", - field=models.DateTimeField(auto_now_add=True, db_index=True), - ), - migrations.AlterField( - model_name="instance", - name="date_modified", - field=models.DateTimeField(auto_now=True, db_index=True), - ), - ], - ) - ] diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 942e0c25f3..5cb57c34fb 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -652,10 +652,10 @@ class Instance(models.Model, InstanceBaseClass): survey_type = models.ForeignKey("logger.SurveyType", on_delete=models.PROTECT) # shows when we first received this instance - date_created = models.DateTimeField(auto_now_add=True, db_index=True) + date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" - date_modified = models.DateTimeField(auto_now=True, db_index=True) + date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) @@ -697,6 +697,11 @@ class Instance(models.Model, InstanceBaseClass): class Meta: app_label = "logger" unique_together = ("xform", "uuid") + indexes = [ + models.Index(fields=['date_created']), + models.Index(fields=['date_modified']), + models.Index(fields=['deleted_at']), + ] @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now(), user=None): From 0f372f83f524134d98ce96ca5e7133b1e92f1c15 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 7 Aug 2023 00:57:05 +0300 Subject: [PATCH 5/6] Apply indexing to json fields Signed-off-by: Kipchirchir Sigei --- .../0005_add_date_fields_indexing.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py b/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py index f5af866ac6..b789dd441d 100644 --- a/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py +++ b/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.18 on 2023-03-14 12:22 +# Generated by Django 3.2.20 on 2023-08-06 13:39 from django.db import migrations, models @@ -13,42 +13,49 @@ class Migration(migrations.Migration): operations = [ migrations.SeparateDatabaseAndState( database_operations=[ + # index the json date fields migrations.RunSQL( - sql='CREATE INDEX CONCURRENTLY "logger_inst_date_cr_42899d_idx" ON "logger_instance" ("date_created");', + sql='CREATE INDEX CONCURRENTLY "logger_inst_date_cr_json_42899d_idx" ON "logger_instance" ((json->>\'_date_created\')) WHERE (json->>\'_date_created\') IS NOT NULL;', + reverse_sql='DROP INDEX "logger_inst_date_cr_json_42899d_idx";', + ), + migrations.RunSQL( + sql='CREATE INDEX CONCURRENTLY "logger_inst_date_mo_json_5a1bd3_idx" ON "logger_instance" ((json->>\'_date_modified\')) WHERE (json->>\'_date_modified\') IS NOT NULL;', + reverse_sql='DROP INDEX "logger_inst_date_mo_json_5a1bd3_idx";', + ), + migrations.RunSQL( + sql='CREATE INDEX CONCURRENTLY "logger_inst_deleted_at_json_da31a3_idx" ON "logger_instance" ((json->>\'_deleted_at\')) WHERE (json->>\'_deleted_at\') IS NOT NULL;', + reverse_sql='DROP INDEX "logger_inst_deleted_at_json_da31a3_idx";', + ), + # index model date fields + migrations.RunSQL( + sql='CREATE INDEX "logger_inst_date_cr_42899d_idx" ON "logger_instance" ("date_created");', reverse_sql='DROP INDEX "logger_inst_date_cr_42899d_idx";', ), migrations.RunSQL( - sql='CREATE INDEX CONCURRENTLY "logger_inst_date_mo_5a1bd3_idx" ON "logger_instance" ("date_modified");', + sql='CREATE INDEX "logger_inst_date_mo_5a1bd3_idx" ON "logger_instance" ("date_modified");', reverse_sql='DROP INDEX "logger_inst_date_mo_5a1bd3_idx";', ), migrations.RunSQL( - sql='CREATE INDEX CONCURRENTLY "logger_inst_deleted_da31a3_idx" ON "logger_instance" ("deleted_at");', - reverse_sql='DROP INDEX "logger_inst_deleted_da31a3_idx";', + sql='CREATE INDEX "logger_inst_deleted_at_da31a3_idx" ON "logger_instance" ("deleted_at");', + reverse_sql='DROP INDEX "logger_inst_deleted_at_da31a3_idx";', ), ], state_operations=[ - migrations.AddIndex( + migrations.AlterField( model_name="instance", - index=models.Index( - fields=["date_created"], name="logger_inst_date_cr_42899d_idx" - ), + name="date_created", + field=models.DateTimeField(auto_now_add=True, db_index=True), ), - migrations.AddIndex( + migrations.AlterField( model_name="instance", - index=models.Index( - fields=["date_modified"], name="logger_inst_date_mo_5a1bd3_idx" - ), + name="date_modified", + field=models.DateTimeField(auto_now=True, db_index=True), ), - migrations.AddIndex( + migrations.AlterField( model_name="instance", - index=models.Index( - fields=["deleted_at"], name="logger_inst_deleted_da31a3_idx" - ), + name="deleted_at", + field=models.DateTimeField(auto_now=True, db_index=True), ), ], ) ] - - - - From 4ad42526cfdd5e727463c138dba304b3a16029a3 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 7 Aug 2023 11:14:52 +0300 Subject: [PATCH 6/6] Fix conflicting migrations Signed-off-by: Kipchirchir Sigei --- ...ng.py => 0008_add_date_fields_indexing.py} | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) rename onadata/apps/logger/migrations/{0005_add_date_fields_indexing.py => 0008_add_date_fields_indexing.py} (65%) diff --git a/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py b/onadata/apps/logger/migrations/0008_add_date_fields_indexing.py similarity index 65% rename from onadata/apps/logger/migrations/0005_add_date_fields_indexing.py rename to onadata/apps/logger/migrations/0008_add_date_fields_indexing.py index b789dd441d..8c933ccc92 100644 --- a/onadata/apps/logger/migrations/0005_add_date_fields_indexing.py +++ b/onadata/apps/logger/migrations/0008_add_date_fields_indexing.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.20 on 2023-08-06 13:39 +# Generated by Django 3.2.20 on 2023-08-07 08:11 from django.db import migrations, models @@ -7,7 +7,7 @@ class Migration(migrations.Migration): atomic = False dependencies = [ - ('logger', '0004_update_instance_geoms'), + ('logger', '0007_alter_projectinvitation_unique_together'), ] operations = [ @@ -28,33 +28,36 @@ class Migration(migrations.Migration): ), # index model date fields migrations.RunSQL( - sql='CREATE INDEX "logger_inst_date_cr_42899d_idx" ON "logger_instance" ("date_created");', + sql='CREATE INDEX CONCURRENTLY "logger_inst_date_cr_42899d_idx" ON "logger_instance" ("date_created");', reverse_sql='DROP INDEX "logger_inst_date_cr_42899d_idx";', ), migrations.RunSQL( - sql='CREATE INDEX "logger_inst_date_mo_5a1bd3_idx" ON "logger_instance" ("date_modified");', + sql='CREATE INDEX CONCURRENTLY "logger_inst_date_mo_5a1bd3_idx" ON "logger_instance" ("date_modified");', reverse_sql='DROP INDEX "logger_inst_date_mo_5a1bd3_idx";', ), migrations.RunSQL( - sql='CREATE INDEX "logger_inst_deleted_at_da31a3_idx" ON "logger_instance" ("deleted_at");', + sql='CREATE INDEX CONCURRENTLY "logger_inst_deleted_at_da31a3_idx" ON "logger_instance" ("deleted_at");', reverse_sql='DROP INDEX "logger_inst_deleted_at_da31a3_idx";', ), ], state_operations=[ - migrations.AlterField( + migrations.AddIndex( model_name="instance", - name="date_created", - field=models.DateTimeField(auto_now_add=True, db_index=True), + index=models.Index( + fields=["date_created"], name="logger_inst_date_cr_42899d_idx" + ), ), - migrations.AlterField( + migrations.AddIndex( model_name="instance", - name="date_modified", - field=models.DateTimeField(auto_now=True, db_index=True), + index=models.Index( + fields=["date_modified"], name="logger_inst_date_mo_5a1bd3_idx" + ), ), - migrations.AlterField( + migrations.AddIndex( model_name="instance", - name="deleted_at", - field=models.DateTimeField(auto_now=True, db_index=True), + index=models.Index( + fields=["deleted_at"], name="logger_inst_deleted_at_da31a3_idx" + ), ), ], )