Merge pull request #42 from RockefellerArchiveCenter/dashboard-cleanup

Add services to clean up archivematica dashboard

fixtures/cassettes/archivematica_cleanup.json

@@ -0,0 +1,111 @@
+{
+    "version": 1,
+    "interactions": [
+        {
+            "request": {
+                "method": "GET",
+                "uri": "http://archivematica-dashboard:8000/api/transfer/completed",
+                "body": null,
+                "headers": {
+                    "User-Agent": [
+                        "python-requests/2.20.0"
+                    ],
+                    "Accept-Encoding": [
+                        "gzip, deflate"
+                    ],
+                    "Accept": [
+                        "*/*"
+                    ],
+                    "Connection": [
+                        "keep-alive"
+                    ]
+                }
+            },
+            "response": {
+                "status": {
+                    "code": 200,
+                    "message": "OK"
+                },
+                "headers": {
+                    "Vary": [
+                        "Accept-Language, Cookie"
+                    ],
+                    "Content-Language": [
+                        "en"
+                    ],
+                    "Content-Type": [
+                        "application/json"
+                    ],
+                    "Connection": [
+                        "keep-alive"
+                    ],
+                    "Server": [
+                        "gunicorn/19.9.0"
+                    ],
+                    "Date": [
+                        "Fri, 21 Dec 2018 00:33:03 GMT"
+                    ],
+                    "Transfer-Encoding": [
+                        "chunked"
+                    ]
+                },
+                "body": {
+                    "string": "{\"message\": \"Fetched completed transfers successfully.\", \"results\": []}"
+                }
+            }
+        },
+        {
+            "request": {
+                "method": "GET",
+                "uri": "http://archivematica-dashboard:8000/api/ingest/completed",
+                "body": null,
+                "headers": {
+                    "User-Agent": [
+                        "python-requests/2.20.0"
+                    ],
+                    "Accept-Encoding": [
+                        "gzip, deflate"
+                    ],
+                    "Accept": [
+                        "*/*"
+                    ],
+                    "Connection": [
+                        "keep-alive"
+                    ]
+                }
+            },
+            "response": {
+                "status": {
+                    "code": 200,
+                    "message": "OK"
+                },
+                "headers": {
+                    "Vary": [
+                        "Accept-Language, Cookie"
+                    ],
+                    "Content-Language": [
+                        "en"
+                    ],
+                    "Content-Type": [
+                        "application/json"
+                    ],
+                    "Connection": [
+                        "keep-alive"
+                    ],
+                    "Server": [
+                        "gunicorn/19.9.0"
+                    ],
+                    "Date": [
+                        "Fri, 21 Dec 2018 00:33:03 GMT"
+                    ],
+                    "Transfer-Encoding": [
+                        "chunked"
+                    ]
+                },
+                "body": {
+                    "string": "{\"message\": \"Fetched completed ingests successfully.\", \"results\": []}"
+                }
+            }
+        }
+    ]
+}

fornax/urls.py

@@ -15,7 +15,7 @@
 """
 from django.conf.urls import url
 from django.urls import include
-from sip_assembly.views import SIPViewSet, SIPAssemblyView, StartTransferView, ApproveTransferView, CleanupRoutineView, CleanupRequestView
+from sip_assembly.views import *
 from rest_framework import routers
 from drf_yasg.views import get_schema_view
 from drf_yasg import openapi
@@ -39,6 +39,8 @@
     url(r'^assemble/', SIPAssemblyView.as_view(), name="assemble-sip"),
     url(r'^start/', StartTransferView.as_view(), name="start-transfer"),
     url(r'^approve/', ApproveTransferView.as_view(), name="approve-transfer"),
+    url(r'^remove-transfers/', RemoveCompletedTransfersView.as_view(), name="remove-transfers"),
+    url(r'^remove-ingests/', RemoveCompletedIngestsView.as_view(), name="remove-ingests"),
     url(r'^cleanup/', CleanupRoutineView.as_view(), name="cleanup"),
     url(r'^request-cleanup/', CleanupRequestView.as_view(), name="request-cleanup"),
     url(r'^status/', include('health_check.api.urls')),

sip_assembly/assemblers.py

@@ -126,6 +126,22 @@ def approve_transfer(self):
         else:
             return "No transfers to approve."
 
+    def remove_completed_transfers(self):
+        """Removes completed transfers from Archivematica dashboard."""
+        try:
+            total = self.client.send_transfer_cleanup_request()
+            return "{} transfers removed from dashboard".format(total)
+        except Exception as e:
+            raise SIPActionError("Error removing transfers from Archivematica dashboard: {}".format(e))
+
+    def remove_completed_ingests(self):
+        """Removes completed transfers from Archivematica dashboard."""
+        try:
+            total = self.client.send_ingest_cleanup_request()
+            return "{} ingests removed from dashboard".format(total)
+        except Exception as e:
+            raise SIPActionError("Error removing ingests from Archivematica dashboard: {}".format(e))
+
 
 class CleanupRequester:
     def __init__(self, url):

sip_assembly/clients.py

@@ -22,7 +22,7 @@ def retrieve(self, uri, *args, **kwargs):
             raise ArchivematicaClientException("Could not return a valid response for {}".format(full_url))
 
     def send_start_transfer_request(self, sip):
-        """Starts and approves transfer in Archivematica."""
+        """Starts a transfer in Archivematica."""
         basepath = "/home/{}.tar.gz".format(sip.bag_identifier)
         full_url = join(self.baseurl, 'transfer/start_transfer/')
         bagpaths = "{}:{}".format(self.location_uuid, basepath)
@@ -34,8 +34,29 @@ def send_start_transfer_request(self, sip):
             raise ArchivematicaClientException(message)
 
     def send_approve_transfer_request(self, sip):
+        """Approves a transfer in Archivematica."""
         approve_transfer = requests.post(join(self.baseurl, 'transfer/approve_transfer/'),
                                          headers=self.headers,
                                          data={'type': 'zipped bag', 'directory': '{}.tar.gz'.format(sip.bag_identifier)})
         if approve_transfer.status_code != 200:
             raise ArchivematicaClientException(approve_transfer.json()['message'])
+
+    def send_ingest_cleanup_request(self):
+        """Removes completed ingests."""
+        completed = self.retrieve('ingest/completed').json()
+        for uuid in completed['results']:
+            full_url = join(self.baseurl, 'ingest/{}/delete/'.format(uuid))
+            resp = requests.delete(full_url, headers=self.headers).json()
+            if not resp['removed']:
+                raise ArchivematicaClientException("Error removing ingest {}".format(uuid))
+        return len(completed['results'])
+
+    def send_transfer_cleanup_request(self):
+        """Removes completed transfers."""
+        completed = self.retrieve('transfer/completed').json()
+        for uuid in completed['results']:
+            full_url = join(self.baseurl, 'transfer/{}/delete/'.format(uuid))
+            resp = requests.delete(full_url, headers=self.headers).json()
+            if not resp['removed']:
+                raise ArchivematicaClientException("Error removing transfer {}".format(uuid))
+        return len(completed['results'])

sip_assembly/tests.py

@@ -13,7 +13,7 @@
 from fornax import settings
 from sip_assembly.assemblers import SIPAssembler, CleanupRoutine, CleanupRequester
 from sip_assembly.models import SIP
-from sip_assembly.views import SIPViewSet, SIPAssemblyView, StartTransferView, ApproveTransferView, CleanupRoutineView, CleanupRequestView
+from sip_assembly.views import *
 
 data_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'json')
 bag_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'bags')
@@ -77,6 +77,15 @@ def archivematica_views(self):
             request = self.factory.post(reverse('start-transfer'))
             response = ApproveTransferView.as_view()(request)
             self.assertEqual(response.status_code, 200, "Wrong HTTP code")
+        with assembly_vcr.use_cassette('archivematica_cleanup.json'):
+            print('*** Cleaning up transfers ***')
+            request = self.factory.post(reverse('remove-transfers'))
+            response = RemoveCompletedTransfersView.as_view()(request)
+            self.assertEqual(response.status_code, 200, "Wrong HTTP code")
+            print('*** Cleaning up ingests ***')
+            request = self.factory.post(reverse('remove-ingests'))
+            response = RemoveCompletedIngestsView.as_view()(request)
+            self.assertEqual(response.status_code, 200, "Wrong HTTP code")
 
     def request_cleanup(self):
         print('*** Requesting cleanup ***')

sip_assembly/views.py

@@ -89,6 +89,30 @@ def post(self, request):
             return Response({"detail": str(e)}, status=500)
 
 
+class RemoveCompletedTransfersView(APIView):
+    """Removes completed transfers from Archivematica dashboard. Accepts POST requests only."""
+
+    def post(self, request):
+        log = logger.new(transaction_id=str(uuid4()))
+        try:
+            message = SIPActions().remove_completed_transfers()
+            return Response({"detail": message}, status=200)
+        except Exception as e:
+            return Response({"detail": str(e)}, status=500)
+
+
+class RemoveCompletedIngestsView(APIView):
+    """Removes completed ingests from Archivematica dashboard. Accepts POST requests only."""
+
+    def post(self, request):
+        log = logger.new(transaction_id=str(uuid4()))
+        try:
+            message = SIPActions().remove_completed_ingests()
+            return Response({"detail": message}, status=200)
+        except Exception as e:
+            return Response({"detail": str(e)}, status=500)
+
+
 class CleanupRequestView(APIView):
     """Sends request to previous microservice to clean up source directory."""