Merge pull request #48 from kartoza/feat-validate-input-layer

add task to move input layer directory when privacy type is changed

django_project/cplus_api/admin.py

@@ -3,9 +3,11 @@
 from cplus_api.models.scenario import ScenarioTask
 from cplus_api.models.layer import InputLayer, OutputLayer, MultipartUpload
 from cplus_api.models.profile import UserProfile, UserRoleType
+from cplus_api.tasks.verify_input_layer import verify_input_layer
 
 
 def cancel_scenario_task(modeladmin, request, queryset):
+    """Cancel scenario task action."""
     for scenario_task in queryset:
         if scenario_task.task_id:
             cancel_task(scenario_task.task_id)
@@ -16,6 +18,17 @@ def cancel_scenario_task(modeladmin, request, queryset):
     )
 
 
+def trigger_verify_input_layer(modeladmin, request, queryset):
+    """Trigger verify input layer in the background."""
+    for input_layer in queryset:
+        verify_input_layer.delay(input_layer.id)
+    modeladmin.message_user(
+        request,
+        'Tasks will be run in the background!',
+        messages.INFO
+    )
+
+
 class ScenarioTaskAdmin(admin.ModelAdmin):
     list_display = ('scenario_name', 'uuid', 'submitted_by', 'task_id',
                     'status', 'progress', 'started_at', 'finished_at',
@@ -36,6 +49,8 @@ class InputLayerAdmin(admin.ModelAdmin):
                     'size', 'component_type', 'privacy_type')
     search_fields = ['name', 'uuid']
     list_filter = ["layer_type", "owner", "component_type", "privacy_type"]
+    readonly_fields = ['layer_type', 'component_type', 'uuid']
+    actions = [trigger_verify_input_layer]
 
 
 class OutputLayerAdmin(admin.ModelAdmin):

django_project/cplus_api/models/layer.py

@@ -1,20 +1,27 @@
 import os
 import uuid
+import shutil
 from zipfile import ZipFile
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.utils import timezone
 from django.core.files.storage import storages, FileSystemStorage
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+
+COMMON_LAYERS_DIR = 'common_layers'
+INTERNAL_LAYERS_DIR = 'internal_layers'
 
 
 def input_layer_dir_path(instance, filename):
     """Return upload directory path for Input Layer."""
     file_path = f'{str(instance.owner.pk)}/'
     if instance.privacy_type == InputLayer.PrivacyTypes.COMMON:
-        file_path = 'common_layers/'
+        file_path = f'{COMMON_LAYERS_DIR}/'
     if instance.privacy_type == InputLayer.PrivacyTypes.INTERNAL:
-        file_path = 'internal_layers/'
+        file_path = f'{INTERNAL_LAYERS_DIR}/'
     file_path = file_path + f'{instance.component_type}/' + filename
     return file_path
 
@@ -164,6 +171,53 @@ def is_available(self):
             return False
         return self.file.storage.exists(self.file.name)
 
+    def is_in_correct_directory(self):
+        layer_path = self.file.name
+        prefix_path = f'{str(self.owner.pk)}/'
+        if self.privacy_type == InputLayer.PrivacyTypes.COMMON:
+            prefix_path = f'{COMMON_LAYERS_DIR}/'
+        elif self.privacy_type == InputLayer.PrivacyTypes.INTERNAL:
+            prefix_path = f'{INTERNAL_LAYERS_DIR}/'
+        return layer_path.startswith(prefix_path)
+
+    def fix_layer_metadata(self):
+        if not self.is_available():
+            return
+        self.size = self.file.size
+        self.save(update_fields=['size'])
+        if self.is_in_correct_directory():
+            return
+        old_path = self.file.name
+        correct_path = input_layer_dir_path(self, self.name)
+        storage = select_input_layer_storage()
+        if isinstance(storage, FileSystemStorage):
+            full_correct_path = os.path.join(storage.location, correct_path)
+            dirname = os.path.split(full_correct_path)[0]
+            os.makedirs(dirname, exist_ok=True)
+            shutil.move(
+                os.path.join(storage.location, old_path),
+                full_correct_path,
+            )
+        else:
+            boto3_client = storage.connection.meta.client
+            copy_source = {
+                'Bucket': storage.bucket_name,
+                'Key': old_path
+            }
+            boto3_client.copy(copy_source, storage.bucket_name, correct_path)
+            boto3_client.delete_object(
+                Bucket=storage.bucket_name, Key=old_path)
+        self.file.name = correct_path
+        self.save(update_fields=['file'])
+
+
+@receiver(post_save, sender=InputLayer)
+def input_layer_post_save(sender, instance: InputLayer,
+                          created, *args, **kwargs):
+    from cplus_api.tasks.verify_input_layer import verify_input_layer
+    if not created:
+        verify_input_layer.delay(instance.id)
+
 
 class OutputLayer(BaseLayer):
 

django_project/cplus_api/tasks/__init__.py

@@ -1,3 +1,4 @@
 from .cleaner import *  # noqa
 from .runner import *  # noqa
 from .remove_layers import *  # noqa
+from .verify_input_layer import *  # noqa

django_project/cplus_api/tasks/verify_input_layer.py

@@ -0,0 +1,24 @@
+import logging
+from celery import shared_task
+from cplus_api.models import (
+    InputLayer
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(name="verify_input_layer")
+def verify_input_layer(layer_id):
+    """
+    Verify input layer: directory + size.
+    """
+    layer = InputLayer.objects.get(id=layer_id)
+    layer.fix_layer_metadata()
+    if layer.is_available():
+        logger.info(
+            f'Layer {layer.uuid} is stored in {layer.file.name} '
+            f'with size {layer.size}'
+        )
+    else:
+        logger.warn(f'Layer {layer.uuid} is not available!')

django_project/cplus_api/tests/common.py

@@ -6,13 +6,18 @@
 from botocore.exceptions import ClientError
 from collections import OrderedDict
 from django.contrib.contenttypes.models import ContentType
+from django.db.models.signals import post_save
 from django.test import TestCase, TransactionTestCase
 from django.core.files.uploadedfile import SimpleUploadedFile
 from rest_framework.test import APIRequestFactory
 from cplus_api.models.profile import UserRoleType
 from cplus_api.tests.factories import UserF
 from django.core.files.storage import storages
-from cplus_api.models.layer import InputLayer, OutputLayer
+from cplus_api.models.layer import (
+    InputLayer,
+    OutputLayer,
+    input_layer_post_save
+)
 
 
 class DummyTask:
@@ -118,6 +123,8 @@ def init_test_data(self):
         )
         self.scenario_task_ct = ContentType.objects.get(
             app_label="cplus_api", model="scenariotask")
+        # disable input layer post save
+        post_save.disconnect(input_layer_post_save, sender=InputLayer)
 
     def cleanup(self):
         """Delete storage used in default and minio."""

django_project/cplus_api/tests/test_model_layer.py

@@ -1,14 +1,19 @@
 import os
 import tempfile
+import mock
+from django.test import Client
+from rest_framework import status
+from django.urls import reverse
 from django.contrib.auth.models import User
 from core.settings.utils import absolute_path
 from cplus_api.tests.factories import InputLayerF, OutputLayerF
 from cplus_api.models.layer import (
     input_layer_dir_path,
     output_layer_dir_path,
-    InputLayer,
+    InputLayer
 )
-from cplus_api.tests.common import BaseAPIViewTransactionTest
+from cplus_api.tasks.verify_input_layer import verify_input_layer
+from cplus_api.tests.common import BaseAPIViewTransactionTest, mocked_process
 
 
 class TestModelLayer(BaseAPIViewTransactionTest):
@@ -130,3 +135,87 @@ def test_download_to_working_directory(self):
         tmp_dir = tempfile.mkdtemp()
         file_path = input_layer_4.download_to_working_directory(tmp_dir)
         self.assertIsNone(file_path)
+
+    def test_is_in_correct_directory(self):
+        input_layer = InputLayerF.create()
+        file_path = absolute_path(
+            'cplus_api', 'tests', 'data',
+            'models', 'test_model_1.tif'
+        )
+        self.store_layer_file(input_layer, file_path)
+        self.assertTrue(input_layer.is_in_correct_directory())
+        input_layer.privacy_type = InputLayer.PrivacyTypes.COMMON
+        input_layer.save()
+        self.assertFalse(input_layer.is_in_correct_directory())
+        input_layer.privacy_type = InputLayer.PrivacyTypes.INTERNAL
+        input_layer.save()
+        self.assertFalse(input_layer.is_in_correct_directory())
+
+    def test_fix_layer_metadata(self):
+        input_layer = InputLayerF.create(
+            name='test_model_fix_1.tif'
+        )
+        input_layer.fix_layer_metadata()
+        self.assertFalse(input_layer.is_available())
+        self.assertEqual(input_layer.size, 0)
+        file_path = absolute_path(
+            'cplus_api', 'tests', 'data',
+            'models', 'test_model_1.tif'
+        )
+        file_size = os.stat(file_path).st_size
+        self.store_layer_file(input_layer, file_path, input_layer.name)
+        input_layer.refresh_from_db()
+        self.assertTrue(input_layer.is_available())
+        input_layer.fix_layer_metadata()
+        self.assertEqual(input_layer.size, file_size)
+        input_layer.privacy_type = InputLayer.PrivacyTypes.COMMON
+        input_layer.save()
+        self.assertFalse(input_layer.is_in_correct_directory())
+        input_layer.fix_layer_metadata()
+        input_layer.refresh_from_db()
+        self.assertTrue(input_layer.is_in_correct_directory())
+
+    def test_verify_input_layer(self):
+        input_layer = InputLayerF.create(
+            name='test_model_verify_1.tif',
+            privacy_type=InputLayer.PrivacyTypes.COMMON
+        )
+        file_path = absolute_path(
+            'cplus_api', 'tests', 'data',
+            'models', 'test_model_1.tif'
+        )
+        file_size = os.stat(file_path).st_size
+        self.store_layer_file(input_layer, file_path, input_layer.name)
+        input_layer.refresh_from_db()
+        self.assertTrue(input_layer.is_available())
+        input_layer.privacy_type = InputLayer.PrivacyTypes.PRIVATE
+        input_layer.save()
+        verify_input_layer(input_layer.id)
+        input_layer.refresh_from_db()
+        self.assertTrue(input_layer.is_in_correct_directory())
+        self.assertEqual(input_layer.size, file_size)
+
+    @mock.patch('cplus_api.admin.verify_input_layer.delay')
+    def test_trigger_verify_input_layer(self, mocked_process_param):
+        mocked_process_param.side_effect = mocked_process
+        input_layer = InputLayerF.create(
+            name='test_model_verify_1.tif',
+            privacy_type=InputLayer.PrivacyTypes.COMMON
+        )
+        file_path = absolute_path(
+            'cplus_api', 'tests', 'data',
+            'models', 'test_model_1.tif'
+        )
+        self.store_layer_file(input_layer, file_path, input_layer.name)
+        client = Client()
+        client.force_login(self.superuser)
+        response = client.post(
+            reverse('admin:cplus_api_inputlayer_changelist'),
+            {
+                'action': 'trigger_verify_input_layer',
+                '_selected_action': [input_layer.id]
+            },
+            follow=True
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        mocked_process_param.assert_called_once()