Updates (#144)

* Add trigger to refresh materialized when organisation name changed

* Add first and last measurements

* Add measurement type on model

* Add measurement type to uploader

* Update generate wells cache with filtering generator

* Update wells form to accomodate measurement type

* Able to make hubeau harvester run in paralel

* Fix error when doing updating well

admin/well.py

@@ -1,12 +1,14 @@
 from django.contrib import admin
 from django.contrib.auth import get_user_model
+from django.db import connections
 from django.urls import reverse
 from django.utils.html import format_html
 
 from gwml2.models.well import (
     Well, WellDocument,
     WellQualityMeasurement, WellYieldMeasurement, WellLevelMeasurement
 )
+from gwml2.models.well_materialized_view import MaterializedViewWell
 
 User = get_user_model()
 
@@ -84,6 +86,32 @@ def last_edited_by_user(self, obj):
 admin.site.register(Well, WellAdmin)
 
 
+def refresh(modeladmin, request, queryset):
+    """Refresh materialized view."""
+    with connections['gwml2'].cursor() as cursor:
+        cursor.execute('REFRESH MATERIALIZED VIEW mv_well_ggmn;')
+        cursor.execute('REFRESH MATERIALIZED VIEW mv_well;')
+
+
+@admin.register(MaterializedViewWell)
+class MaterializedViewWellAdmin(admin.ModelAdmin):
+    list_display = (
+        'ggis_uid', 'id', 'name', 'organisation',
+        'country', 'first_time_measurement', 'last_time_measurement',
+        'number_of_measurements_level', 'number_of_measurements_quality',
+        'is_groundwater_level', 'is_groundwater_quality',
+        'link',
+    )
+    list_filter = (
+        'first_time_measurement', 'last_time_measurement'
+    )
+    search_fields = ('ggis_uid', 'name')
+    actions = (refresh,)
+
+    def link(self, obj):
+        return format_html(obj.detail)
+
+
 class MeasurementAdmin(admin.ModelAdmin):
     list_display = (
         'well', 'time', 'parameter', 'methodology', 'value',

forms/well/general_information.py

@@ -25,7 +25,8 @@ class Meta:
             'ggis_uid', 'original_id', 'location', 'name', 'feature_type',
             'purpose', 'status', 'country', 'address',
             'ground_surface_elevation', 'top_borehole_elevation', 'photo',
-            'description', 'glo_90m_elevation'
+            'description', 'glo_90m_elevation',
+            'is_groundwater_level', 'is_groundwater_quality'
         )
         widgets = {
             'ground_surface_elevation': QuantityInput(unit_group='length'),

harvesters/harvester/hubeau.py

@@ -36,6 +36,9 @@ class Hubeau(BaseHarvester):
     current_idx = 0
     proceed = False
 
+    # Filter the index by using - separator
+    index_filter = 'index-filter'
+
     def __init__(
             self, harvester: Harvester, replace: bool = False,
             original_id: str = None
@@ -53,6 +56,21 @@ def _process(self):
         except Exception:
             pass
 
+        self.indexes = None
+        try:
+            indexes = self.attributes['index-filter'].split('-')
+            self.indexes = [
+                int(indexes[0]), int(indexes[1])
+            ]
+        except Exception:
+            pass
+
+        if self.indexes:
+            if self.indexes[0] > self.indexes[0]:
+                raise Exception(
+                    'First index on index-filter is greater than last index'
+                )
+
         # Check last code
         self.last_code = self.attributes.get(self.last_code_key, None)
         if not self.last_code:
@@ -127,6 +145,13 @@ def _process_stations(self, url: str):
 
             for station in stations:
                 self.current_idx += 1
+                if self.indexes:
+                    if (
+                            self.current_idx < self.indexes[0] or
+                            self.current_idx > self.indexes[1]
+                    ):
+                        continue
+
                 original_id = station[self.original_id_key]
                 self._update(
                     f'[{self.current_idx}/{self.count}] '
@@ -214,7 +239,8 @@ def _process_stations(self, url: str):
                         self.post_processing_well(
                             well, generate_country_cache=False
                         )
-                        self.countries.append(well.country.code)
+                        if well.country:
+                            self.countries.append(well.country.code)
                 except (
                         Well.DoesNotExist, requests.exceptions.ConnectionError
                 ):

harvesters/models/harvester.py

@@ -12,15 +12,13 @@ class Harvester(models.Model):
     """
     harvester_class = models.CharField(
         max_length=100,
-        unique=True,
         help_text=_(
             "The type of harvester that will be used."
             "Use class with full package.")
     )
     name = models.CharField(
         _('Name'),
         max_length=100,
-        unique=True,
         help_text=_("Name of harvester that.")
     )
     description = models.TextField(
@@ -71,6 +69,7 @@ class Harvester(models.Model):
 
     class Meta:
         db_table = 'harvester'
+        unique_together = ('harvester_class', 'name')
 
     def __str__(self):
         return self.harvester_class

management/commands/generate_data_wells_cache.py

@@ -38,6 +38,12 @@ def add_arguments(self, parser):
             dest='force',
             help='Force to recreate the data'
         )
+        parser.add_argument(
+            '-generators',
+            '--generators',
+            dest='generators',
+            help="Filter by generators : ['general_information','hydrogeology','management','drilling_and_construction','monitor'] in comma separator"
+        )
 
     def handle(self, *args, **options):
         id = options.get('id', False)
@@ -57,6 +63,8 @@ def handle(self, *args, **options):
         if country_code:
             wells = wells.filter(country__code=country_code)
 
+        generators = options.get('generators', None)
+
         # Regenerate cache
         countries = []
         organisations = []
@@ -66,8 +74,10 @@ def handle(self, *args, **options):
             well = Well.objects.get(id=id)
             print(f'----- {idx}/{count} - {well.id} -----')
             generate_data_well_cache(
-                well.id, force_regenerate=force, generate_country_cache=False,
-                generate_organisation_cache=False
+                well.id, force_regenerate=force,
+                generate_country_cache=False,
+                generate_organisation_cache=False,
+                generators=generators.split(',') if generators else None
             )
 
             # Save the country code

management/commands/update_measurement_type.py

@@ -0,0 +1,44 @@
+from django.core.management.base import BaseCommand
+from django.db.models.signals import post_save
+
+from gwml2.models.well import Well
+from gwml2.utilities import temp_disconnect_signal
+
+
+class Command(BaseCommand):
+    """ Update all fixtures
+    """
+    args = ''
+    help = 'Update measurement type of well.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-from',
+            '--from',
+            default='0',
+            dest='from',
+            help='From'
+        )
+
+    def handle(self, *args, **options):
+        from gwml2.signals.well import update_well
+        _from = int(options['from'])
+        with temp_disconnect_signal(
+                signal=post_save,
+                receiver=update_well,
+                sender=Well
+        ):
+            for idx, well in enumerate(Well.objects.all()):
+                if idx + 1 < _from:
+                    continue
+                if well.is_groundwater_level is None:
+                    well.is_groundwater_level = (
+                        'no' if well.welllevelmeasurement_set.count() == 0
+                        else 'yes'
+                    )
+                if well.is_groundwater_quality is None:
+                    well.is_groundwater_quality = (
+                        'no' if well.wellqualitymeasurement_set.count() == 0
+                        else 'yes'
+                    )
+                well.save()

migrations/0085_view_well.py

@@ -1,9 +1,12 @@
 # Generated by Django 2.2.12 on 2020-07-03 09:03
 
-from gwml2.migrations.sql.view_well_migrations import ViewWellMigration
+from django.db import migrations
 
 
-class Migration(ViewWellMigration):
+class Migration(migrations.Migration):
     dependencies = [
         ('gwml2', '0084_view_well'),
     ]
+
+    operations = [
+    ]

migrations/0091_auto_20250124_0657.py

@@ -0,0 +1,60 @@
+# Generated by Django 3.2.20 on 2025-01-24 06:57
+
+import django.contrib.gis.db.models.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gwml2', '0090_auto_20250103_0752'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MaterializedViewWell',
+            fields=[
+                ('id', models.IntegerField(primary_key=True, serialize=False)),
+                ('ggis_uid', models.CharField(help_text='Organisation + id.', max_length=512)),
+                ('original_id', models.CharField(help_text='As recorded in the original database.', max_length=512)),
+                ('name', models.CharField(blank=True, max_length=512, null=True)),
+                ('feature_type', models.CharField(blank=True, max_length=512, null=True)),
+                ('purpose', models.CharField(blank=True, max_length=512, null=True)),
+                ('status', models.CharField(blank=True, max_length=512, null=True)),
+                ('organisation', models.CharField(blank=True, max_length=512, null=True)),
+                ('organisation_id', models.IntegerField()),
+                ('country', models.CharField(blank=True, max_length=512, null=True)),
+                ('year_of_drilling', models.PositiveIntegerField(blank=True, null=True)),
+                ('aquifer_name', models.CharField(blank=True, max_length=512, null=True)),
+                ('aquifer_type', models.CharField(blank=True, max_length=512, null=True)),
+                ('manager', models.CharField(blank=True, max_length=512, null=True)),
+                ('detail', models.CharField(blank=True, max_length=512, null=True)),
+                ('location', django.contrib.gis.db.models.fields.PointField(srid=4326)),
+                ('created_at', models.DateTimeField(blank=True, null=True)),
+                ('created_by', models.IntegerField(blank=True, null=True)),
+                ('last_edited_at', models.DateTimeField(blank=True, null=True)),
+                ('last_edited_by', models.IntegerField(blank=True, null=True)),
+                ('number_of_measurements_level', models.IntegerField()),
+                ('number_of_measurements_quality', models.IntegerField()),
+                ('first_time_measurement', models.DateTimeField(blank=True, null=True)),
+                ('last_time_measurement', models.DateTimeField(blank=True, null=True)),
+                ('is_groundwater_level', models.CharField(blank=True, max_length=8, null=True)),
+                ('is_groundwater_quality', models.CharField(blank=True, max_length=8, null=True)),
+            ],
+            options={
+                'db_table': 'mv_well',
+                'ordering': ['original_id'],
+                'managed': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='well',
+            name='is_groundwater_level',
+            field=models.CharField(blank=True, choices=[('yes', 'yes'), ('no', 'no')], max_length=8, null=True),
+        ),
+        migrations.AddField(
+            model_name='well',
+            name='is_groundwater_quality',
+            field=models.CharField(blank=True, choices=[('yes', 'yes'), ('no', 'no')], max_length=8, null=True),
+        ),
+    ]

migrations/0092_view_well.py

@@ -0,0 +1,9 @@
+# Generated by Django 2.2.12 on 2020-07-03 09:03
+
+from gwml2.migrations.sql.view_well_migrations import ViewWellMigration
+
+
+class Migration(ViewWellMigration):
+    dependencies = [
+        ('gwml2', '0091_auto_20250124_0657'),
+    ]

migrations/0093_auto_20250124_0820.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.20 on 2025-01-24 08:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gwml2', '0092_view_well'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='harvester',
+            name='harvester_class',
+            field=models.CharField(help_text='The type of harvester that will be used.Use class with full package.', max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='harvester',
+            name='name',
+            field=models.CharField(help_text='Name of harvester that.', max_length=100, verbose_name='Name'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='harvester',
+            unique_together={('harvester_class', 'name')},
+        ),
+    ]

migrations/sql/gwml/default.sql

@@ -4,6 +4,7 @@ DROP MATERIALIZED VIEW IF EXISTS mv_well_measurement;
 DROP VIEW IF EXISTS vw_well_measurement;
 DROP VIEW IF EXISTS vw_well;
 
+DROP TRIGGER IF EXISTS trigger_country ON country;
 DROP TRIGGER IF EXISTS trigger_organisation ON organisation;
 DROP TRIGGER IF EXISTS trigger_user_uuid ON user_uuid;
 DROP TRIGGER IF EXISTS trigger_well ON well;

migrations/sql/gwml/functions.sql

@@ -10,6 +10,16 @@ END;
 LANGUAGE plpgsql;
 
 CREATE TRIGGER trigger_well
-    AFTER INSERT
+    AFTER INSERT OR UPDATE OF last_edited_at
     ON well
+    FOR EACH ROW EXECUTE PROCEDURE update_mv();
+
+CREATE TRIGGER trigger_organisation
+    AFTER UPDATE OF name
+    ON organisation
+    FOR EACH ROW EXECUTE PROCEDURE update_mv();
+
+CREATE TRIGGER trigger_country
+    AFTER UPDATE OF name
+    ON country
     FOR EACH ROW EXECUTE PROCEDURE update_mv();