Remote ESSArch

ESSArch_Core/WorkflowEngine/models.py

@@ -604,15 +604,17 @@ def reraise(self):
     def create_remote_copy(self, session, host):
         create_remote_task_url = urljoin(host, reverse('processtask-list'))
         params = copy.deepcopy(self.params)
-        params.pop('_options', None)
+        params['storage_object'] = str(params['storage_object']) if params.get('storage_object') is not None else None
         ip_id = str(self.information_package.pk) if self.information_package.pk is not None else None
+        responsible_username = self.responsible.username if self.responsible is not None else None
         data = {
             'id': str(self.pk),
             'name': self.name,
             'args': self.args,
-            'params': self.params,
+            'params': params,
             'eager': self.eager,
             'information_package': ip_id,
+            'responsible': responsible_username,
         }
         r = session.post(create_remote_task_url, json=data, timeout=60)
 
@@ -627,14 +629,16 @@ def create_remote_copy(self, session, host):
     def update_remote_copy(self, session, host):
         update_remote_task_url = urljoin(host, reverse('processtask-detail', args=(str(self.pk),)))
         params = copy.deepcopy(self.params)
-        params.pop('_options', None)
+        params['storage_object'] = str(params['storage_object']) if params.get('storage_object') is not None else None
         ip_id = str(self.information_package.pk) if self.information_package.pk is not None else None
+        responsible_username = self.responsible.username if self.responsible is not None else None
         data = {
             'name': self.name,
             'args': self.args,
-            'params': self.params,
+            'params': params,
             'eager': self.eager,
             'information_package': ip_id,
+            'responsible': responsible_username,
         }
         r = session.patch(update_remote_task_url, json=data, timeout=60)
 

ESSArch_Core/WorkflowEngine/serializers.py

@@ -25,6 +25,7 @@
 import uuid
 
 from celery import states as celery_states
+from django.contrib.auth import get_user_model
 from rest_framework import serializers
 
 from ESSArch_Core.auth.fields import CurrentUsernameDefault
@@ -33,6 +34,8 @@
 from ESSArch_Core.WorkflowEngine.models import ProcessStep, ProcessTask
 from ESSArch_Core.WorkflowEngine.util import get_result
 
+User = get_user_model()
+
 
 class ProcessStepChildrenSerializer(serializers.Serializer):
     url = serializers.SerializerMethodField()
@@ -84,7 +87,7 @@ class ProcessTaskSerializer(serializers.ModelSerializer):
     args = serializers.JSONField(required=False)
     params = serializers.SerializerMethodField()
     responsible = serializers.SlugRelatedField(
-        slug_field='username', read_only=True
+        slug_field='username', queryset=User.objects.all(), required=False,
     )
 
     def get_params(self, obj):

ESSArch_Core/celery/backends/database.py

@@ -101,8 +101,11 @@ def _get_task_meta_for(self, task_id):
 
     @classmethod
     def exception_to_python(cls, exc):
-        """Convert serialized exception to Python exception."""
+        """Convert serialized exception or string to Python exception."""
         if exc:
+            if isinstance(exc, str):
+                exc = Exception(exc)
+
             if not isinstance(exc, BaseException):
                 exc_module = exc.get('exc_module')
                 if exc_module is None:

ESSArch_Core/configuration/serializers.py

@@ -34,6 +34,7 @@
     Site,
     StoragePolicy,
 )
+from ESSArch_Core.exceptions import Conflict
 from ESSArch_Core.storage.models import (
     StorageMethod,
     StorageMethodTargetRelation,
@@ -133,10 +134,80 @@ class Meta:
 
 
 class StoragePolicySerializer(serializers.ModelSerializer):
-    cache_storage = StorageMethodSerializer()
+    cache_storage = StorageMethodSerializer(allow_null=True)
     storage_methods = StorageMethodSerializer(many=True)
     ingest_path = PathSerializer()
 
+    def create_storage_method(self, data):
+        if data is None:
+            return None
+
+        storage_method_target_set_data = data.pop('storage_method_target_relations')
+        storage_method, _ = StorageMethod.objects.update_or_create(
+            id=data['id'],
+            defaults=data
+        )
+
+        for storage_method_target_data in storage_method_target_set_data:
+            storage_target_data = storage_method_target_data.pop('storage_target')
+            storage_target_data.pop('remote_server', None)
+            storage_target, _ = StorageTarget.objects.update_or_create(
+                id=storage_target_data['id'],
+                defaults=storage_target_data
+            )
+            storage_method_target_data['storage_method'] = storage_method
+            storage_method_target_data['storage_target'] = storage_target
+            storage_method_target, _ = StorageMethodTargetRelation.objects.update_or_create(
+                id=storage_method_target_data['id'],
+                defaults=storage_method_target_data
+            )
+
+        return storage_method
+
+    def create(self, validated_data):
+        storage_method_set_data = validated_data.pop('storage_methods')
+        cache_storage_data = validated_data.pop('cache_storage')
+        ingest_path_data = validated_data.pop('ingest_path')
+
+        cache_storage = self.create_storage_method(cache_storage_data)
+        ingest_path, _ = Path.objects.update_or_create(entity=ingest_path_data['entity'], defaults=ingest_path_data)
+
+        validated_data['cache_storage'] = cache_storage
+        validated_data['ingest_path'] = ingest_path
+
+        policy, _ = StoragePolicy.objects.update_or_create(policy_id=validated_data['policy_id'],
+                                                           defaults=validated_data)
+
+        for storage_method_data in storage_method_set_data:
+            storage_method = self.create_storage_method(storage_method_data)
+            policy.storage_methods.add(storage_method)
+            # add to policy, dummy
+
+        return policy
+
+    def update(self, instance, validated_data):
+        storage_method_set_data = validated_data.pop('storage_methods')
+        cache_storage_data = validated_data.pop('cache_storage')
+        ingest_path_data = validated_data.pop('ingest_path')
+
+        cache_storage = self.create_storage_method(cache_storage_data)
+        ingest_path, _ = Path.objects.update_or_create(entity=ingest_path_data['entity'], defaults=ingest_path_data)
+
+        validated_data['cache_storage'] = cache_storage
+        validated_data['ingest_path'] = ingest_path
+
+        for storage_method_data in storage_method_set_data:
+            storage_method = self.create_storage_method(storage_method_data)
+            instance.storage_methods.add(storage_method)
+
+        return super().update(instance, validated_data)
+
+    def validate(self, data):
+        if self.instance is None and StoragePolicy.objects.filter(pk=data.get('id')).exists():
+            raise Conflict('Storage policy already exists')
+
+        return data
+
     class Meta:
         model = StoragePolicy
         fields = (
@@ -153,6 +224,7 @@ class Meta:
         )
         extra_kwargs = {
             'id': {
+                'read_only': False,
                 'validators': [],
             },
             'policy_id': {

ESSArch_Core/frontend/static/frontend/lang/en/import.ts

@@ -18,6 +18,9 @@ export default ($translateProvider: ng.translate.ITranslateProvider) => {
       SA_EXISTS_DESC: 'Submission agreement with same ID already exists. Would you like to overwrite it?',
       SA_IMPORTED: 'Submission agreement "{{name}}" has been imported. \nID: {{id}}',
       SA_IS_PUBLISHED_CANNOT_BE_OVERWRITTEN: 'Submission Agreement {{name}} is Published and can not be overwritten',
+      STORAGEPOLICY_EXISTS: 'Storage policy exists',
+      STORAGEPOLICY_EXISTS_DESC: 'Storage policy with same ID already exists. Would you like to overwrite it?',
+      STORAGEPOLICY_IMPORTED: 'Storage policy: "{{name}}" has been imported. \nID: {{id}}',
     },
   });
 };

ESSArch_Core/frontend/static/frontend/lang/sv/import.ts

@@ -19,6 +19,9 @@ export default ($translateProvider: ng.translate.ITranslateProvider) => {
       SA_IMPORTED: 'Leveransöverenskommelse "{{name}}" har importerats. \nID: {{id}}',
       SA_IS_PUBLISHED_CANNOT_BE_OVERWRITTEN:
         'Leveransöverenskommelse {{name}} är publicerad och kan inte skrivas över',
+      STORAGEPOLICY_EXISTS: 'Lagringsregelverk finns redan',
+      STORAGEPOLICY_EXISTS_DESC: 'Lagringsregelverk med samma ID finns redan. VIll du skriva över den?',
+      STORAGEPOLICY_IMPORTED: 'Lagringsregelverk: "{{name}}" har importerats. \nID: {{id}}',
     },
   });
 };

ESSArch_Core/frontend/static/frontend/scripts/components/ImportComponent.js

@@ -13,6 +13,7 @@ export default {
     'Notifications',
     '$uibModal',
     '$translate',
+    'StoragePolicy',
     controller,
   ],
   controllerAs: 'vm',

ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js

@@ -1,5 +1,5 @@
 export default class ImportCtrl {
-  constructor($q, $rootScope, $scope, $http, IP, Profile, SA, Notifications, $uibModal, $translate) {
+  constructor($q, $rootScope, $scope, $http, IP, Profile, SA, Notifications, $uibModal, $translate, StoragePolicy) {
     const vm = this;
     $scope.angular = angular;
     vm.loadingSas = false;
@@ -70,6 +70,23 @@ export default class ImportCtrl {
                 });
             })
           );
+        } else if (key === 'policy') {
+          promises.push(
+            $http.get(vm.url + '/api/storage-policies/' + sa[key] + '/', {headers: headers}).then(function (response) {
+              const data = response.data;
+              return StoragePolicy.new(data)
+                .$promise.then(function (response) {
+                  return response;
+                })
+                .catch(function (response) {
+                  vm.importingSa = false;
+                  if (response.status == 409) {
+                    storagePolicyExistsModal(data);
+                  }
+                  return response;
+                });
+            })
+          );
         } else {
         }
       }
@@ -198,6 +215,24 @@ export default class ImportCtrl {
       });
       modalInstance.result.then(function (data) {});
     }
+    function storagePolicyExistsModal(profile) {
+      const modalInstance = $uibModal.open({
+        animation: true,
+        ariaLabelledBy: 'modal-title',
+        ariaDescribedBy: 'modal-body',
+        templateUrl: 'static/frontend/views/modals/storagePolicy-exists-modal.html',
+        controller: 'OverwriteModalInstanceCtrl',
+        controllerAs: '$ctrl',
+        resolve: {
+          data: function () {
+            return {
+              profile: profile,
+            };
+          },
+        },
+      });
+      modalInstance.result.then(function (data) {});
+    }
     vm.triggerProfileUpload = function () {
       document.getElementById('profile-upload').click();
     };

ESSArch_Core/frontend/static/frontend/scripts/controllers/OverwriteModalInstanceCtrl.js

@@ -1,5 +1,5 @@
 export default class OverwriteModalInstanceCtrl {
-  constructor($uibModalInstance, data, Profile, SA, Notifications, $translate) {
+  constructor($uibModalInstance, data, Profile, SA, Notifications, $translate, StoragePolicy) {
     const $ctrl = this;
     if (data.file) {
       $ctrl.file = data.file;
@@ -20,6 +20,18 @@ export default class OverwriteModalInstanceCtrl {
         return resource;
       });
     };
+    $ctrl.overwriteStoragePolicy = function () {
+      return StoragePolicy.update($ctrl.profile).$promise.then(function (resource) {
+        Notifications.add($translate.instant('IMPORT.STORAGEPOLICY_IMPORTED', resource), 'success', 5000, {
+          isHtml: true,
+        });
+        $ctrl.data = {
+          status: 'overwritten',
+        };
+        $uibModalInstance.close($ctrl.data);
+        return resource;
+      });
+    };
     $ctrl.overwriteSa = function () {
       return SA.update($ctrl.profile)
         .$promise.then(function (resource) {

ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js

@@ -784,6 +784,7 @@ export default angular
     'SA',
     'Notifications',
     '$translate',
+    'StoragePolicy',
     OverwriteModalInstanceCtrl,
   ])
   .controller('PlaceNodeInArchiveModalInstanceCtrl', [

ESSArch_Core/frontend/static/frontend/scripts/services/storagePolicy.js

@@ -1,3 +1,19 @@
 export default ($resource, appConfig) => {
-  return $resource(appConfig.djangoUrl + 'storage-policies/:id/:action/', {id: '@id'}, {});
+  return $resource(
+    appConfig.djangoUrl + 'storage-policies/:id/:action/',
+    {},
+    {
+      get: {
+        method: 'GET',
+        params: {id: '@id'},
+      },
+      new: {
+        method: 'POST',
+      },
+      update: {
+        method: 'PUT',
+        params: {id: '@id'},
+      },
+    }
+  );
 };

ESSArch_Core/frontend/static/frontend/views/modals/storagePolicy-exists-modal.html

@@ -0,0 +1,19 @@
+<div class="modal-heading">
+  <h3 class="modal-title" id="modal-title">{{ "IMPORT.STORAGEPOLICY_EXISTS" | translate }}</h3>
+</div>
+<form ng-submit="$ctrl.overwriteStoragePolicy()">
+  <div class="modal-body" id="modal-body">
+    <p for="label">{{ "IMPORT.STORAGEPOLICY_EXISTS_DESC" | translate }}</p>
+    <b>{{ "NAME" | translate }}: {{ $ctrl.profile.policy_name }}</b>
+    <br />
+    <b>ID: {{ $ctrl.profile.id }}</b>
+    <br />
+    <b ng-if="$ctrl.profile.policy_id">Policy ID: {{ $ctrl.profile.policy_id }}</b>
+    <br />
+    <b ng-if="$ctrl.profile.ais_project_id">AIS Policy ID: {{ $ctrl.profile.ais_project_id }}</b>
+  </div>
+  <div class="modal-footer">
+    <button class="btn btn-primary" focused="true" type="submit">{{ "OK" | translate }}</button>
+    <button class="btn btn-default" type="button" ng-click="$ctrl.cancel()">{{ "CANCEL" | translate }}</button>
+  </div>
+</form>

ESSArch_Core/ip/models.py

@@ -1716,12 +1716,21 @@ def get_temp_container_aic_xml_path(self):
     @retry(retry=retry_if_exception_type(RequestException), reraise=True, stop=stop_after_attempt(5),
            wait=wait_fixed(60), before_sleep=before_sleep_log(logger, logging.DEBUG))
     def update_remote_ip(self, host, session):
-        from ESSArch_Core.ip.serializers import InformationPackageFromMasterSerializer
+        from ESSArch_Core.ip.serializers import (
+            InformationPackageFromMasterSerializer,
+        )
 
         remote_ip = urljoin(host, reverse('informationpackage-add-from-master'))
         data = InformationPackageFromMasterSerializer(instance=self).data
-        response = session.post(remote_ip, json=data, timeout=10)
-        response.raise_for_status()
+        response = None
+        try:
+            response = session.post(remote_ip, json=data, timeout=10)
+            response.raise_for_status()
+        except RequestException as e:
+            msg = 'Response: {response}, post_url: {post_url}, post_data: {post_data}'.format(
+                response=e.response.text, post_url=remote_ip, post_data=data)
+            logger.error(msg)
+            raise e
 
     @retry(retry=retry_if_exception_type(StorageMediumFull), reraise=True, stop=stop_after_attempt(2),
            wait=wait_fixed(60), before_sleep=before_sleep_log(logger, logging.DEBUG))
@@ -1759,6 +1768,8 @@ def preserve(self, src: list, storage_target, container: bool, task):
                 task.result = remote_data['result']
                 task.traceback = remote_data['traceback']
                 task.exception = remote_data['exception']
+                if task.status == 'SUCCESS':
+                    storage_object = StorageObject.create_from_remote_copy(host, session, task.result)
                 task.save()
 
                 if task.status != celery_states.SUCCESS:
@@ -1773,14 +1784,15 @@ def preserve(self, src: list, storage_target, container: bool, task):
                 task.result = remote_data['result']
                 task.traceback = remote_data['traceback']
                 task.exception = remote_data['exception']
+                if task.status == 'SUCCESS':
+                    storage_object = StorageObject.create_from_remote_copy(host, session, task.result)
                 task.save()
 
                 sleep(5)
 
             if task.status in celery_states.EXCEPTION_STATES:
                 task.reraise()
 
-            storage_object = StorageObject.create_from_remote_copy(host, session, task.result)
         else:
             storage_medium, created = storage_target.get_or_create_storage_medium(qs=qs)