diff --git a/fixtures/cassettes/archivematica_cleanup.json b/fixtures/cassettes/archivematica_cleanup.json new file mode 100644 index 0000000..a236bf9 --- /dev/null +++ b/fixtures/cassettes/archivematica_cleanup.json @@ -0,0 +1,111 @@ +{ + "version": 1, + "interactions": [ + { + "request": { + "method": "GET", + "uri": "http://archivematica-dashboard:8000/api/transfer/completed", + "body": null, + "headers": { + "User-Agent": [ + "python-requests/2.20.0" + ], + "Accept-Encoding": [ + "gzip, deflate" + ], + "Accept": [ + "*/*" + ], + "Connection": [ + "keep-alive" + ] + } + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Vary": [ + "Accept-Language, Cookie" + ], + "Content-Language": [ + "en" + ], + "Content-Type": [ + "application/json" + ], + "Connection": [ + "keep-alive" + ], + "Server": [ + "gunicorn/19.9.0" + ], + "Date": [ + "Fri, 21 Dec 2018 00:33:03 GMT" + ], + "Transfer-Encoding": [ + "chunked" + ] + }, + "body": { + "string": "{\"message\": \"Fetched completed transfers successfully.\", \"results\": []}" + } + } + }, + { + "request": { + "method": "GET", + "uri": "http://archivematica-dashboard:8000/api/ingest/completed", + "body": null, + "headers": { + "User-Agent": [ + "python-requests/2.20.0" + ], + "Accept-Encoding": [ + "gzip, deflate" + ], + "Accept": [ + "*/*" + ], + "Connection": [ + "keep-alive" + ] + } + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Vary": [ + "Accept-Language, Cookie" + ], + "Content-Language": [ + "en" + ], + "Content-Type": [ + "application/json" + ], + "Connection": [ + "keep-alive" + ], + "Server": [ + "gunicorn/19.9.0" + ], + "Date": [ + "Fri, 21 Dec 2018 00:33:03 GMT" + ], + "Transfer-Encoding": [ + "chunked" + ] + }, + "body": { + "string": "{\"message\": \"Fetched completed ingests successfully.\", \"results\": []}" + } + } + } + ] +} \ No newline at end of file diff --git a/fornax/urls.py b/fornax/urls.py index aa188b6..df93a7f 100644 --- a/fornax/urls.py +++ b/fornax/urls.py @@ -15,7 +15,7 @@ """ from django.conf.urls import url from django.urls import include -from sip_assembly.views import SIPViewSet, SIPAssemblyView, StartTransferView, ApproveTransferView, CleanupRoutineView, CleanupRequestView +from sip_assembly.views import * from rest_framework import routers from drf_yasg.views import get_schema_view from drf_yasg import openapi @@ -39,6 +39,8 @@ url(r'^assemble/', SIPAssemblyView.as_view(), name="assemble-sip"), url(r'^start/', StartTransferView.as_view(), name="start-transfer"), url(r'^approve/', ApproveTransferView.as_view(), name="approve-transfer"), + url(r'^remove-transfers/', RemoveCompletedTransfersView.as_view(), name="remove-transfers"), + url(r'^remove-ingests/', RemoveCompletedIngestsView.as_view(), name="remove-ingests"), url(r'^cleanup/', CleanupRoutineView.as_view(), name="cleanup"), url(r'^request-cleanup/', CleanupRequestView.as_view(), name="request-cleanup"), url(r'^status/', include('health_check.api.urls')), diff --git a/sip_assembly/assemblers.py b/sip_assembly/assemblers.py index af1e90e..037ba37 100644 --- a/sip_assembly/assemblers.py +++ b/sip_assembly/assemblers.py @@ -126,6 +126,22 @@ def approve_transfer(self): else: return "No transfers to approve." + def remove_completed_transfers(self): + """Removes completed transfers from Archivematica dashboard.""" + try: + total = self.client.send_transfer_cleanup_request() + return "{} transfers removed from dashboard".format(total) + except Exception as e: + raise SIPActionError("Error removing transfers from Archivematica dashboard: {}".format(e)) + + def remove_completed_ingests(self): + """Removes completed transfers from Archivematica dashboard.""" + try: + total = self.client.send_ingest_cleanup_request() + return "{} ingests removed from dashboard".format(total) + except Exception as e: + raise SIPActionError("Error removing ingests from Archivematica dashboard: {}".format(e)) + class CleanupRequester: def __init__(self, url): diff --git a/sip_assembly/clients.py b/sip_assembly/clients.py index 0f14aaf..29ea9b7 100644 --- a/sip_assembly/clients.py +++ b/sip_assembly/clients.py @@ -22,7 +22,7 @@ def retrieve(self, uri, *args, **kwargs): raise ArchivematicaClientException("Could not return a valid response for {}".format(full_url)) def send_start_transfer_request(self, sip): - """Starts and approves transfer in Archivematica.""" + """Starts a transfer in Archivematica.""" basepath = "/home/{}.tar.gz".format(sip.bag_identifier) full_url = join(self.baseurl, 'transfer/start_transfer/') bagpaths = "{}:{}".format(self.location_uuid, basepath) @@ -34,8 +34,29 @@ def send_start_transfer_request(self, sip): raise ArchivematicaClientException(message) def send_approve_transfer_request(self, sip): + """Approves a transfer in Archivematica.""" approve_transfer = requests.post(join(self.baseurl, 'transfer/approve_transfer/'), headers=self.headers, data={'type': 'zipped bag', 'directory': '{}.tar.gz'.format(sip.bag_identifier)}) if approve_transfer.status_code != 200: raise ArchivematicaClientException(approve_transfer.json()['message']) + + def send_ingest_cleanup_request(self): + """Removes completed ingests.""" + completed = self.retrieve('ingest/completed').json() + for uuid in completed['results']: + full_url = join(self.baseurl, 'ingest/{}/delete/'.format(uuid)) + resp = requests.delete(full_url, headers=self.headers).json() + if not resp['removed']: + raise ArchivematicaClientException("Error removing ingest {}".format(uuid)) + return len(completed['results']) + + def send_transfer_cleanup_request(self): + """Removes completed transfers.""" + completed = self.retrieve('transfer/completed').json() + for uuid in completed['results']: + full_url = join(self.baseurl, 'transfer/{}/delete/'.format(uuid)) + resp = requests.delete(full_url, headers=self.headers).json() + if not resp['removed']: + raise ArchivematicaClientException("Error removing transfer {}".format(uuid)) + return len(completed['results']) diff --git a/sip_assembly/tests.py b/sip_assembly/tests.py index 8260937..67ded8c 100644 --- a/sip_assembly/tests.py +++ b/sip_assembly/tests.py @@ -13,7 +13,7 @@ from fornax import settings from sip_assembly.assemblers import SIPAssembler, CleanupRoutine, CleanupRequester from sip_assembly.models import SIP -from sip_assembly.views import SIPViewSet, SIPAssemblyView, StartTransferView, ApproveTransferView, CleanupRoutineView, CleanupRequestView +from sip_assembly.views import * data_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'json') bag_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'bags') @@ -77,6 +77,15 @@ def archivematica_views(self): request = self.factory.post(reverse('start-transfer')) response = ApproveTransferView.as_view()(request) self.assertEqual(response.status_code, 200, "Wrong HTTP code") + with assembly_vcr.use_cassette('archivematica_cleanup.json'): + print('*** Cleaning up transfers ***') + request = self.factory.post(reverse('remove-transfers')) + response = RemoveCompletedTransfersView.as_view()(request) + self.assertEqual(response.status_code, 200, "Wrong HTTP code") + print('*** Cleaning up ingests ***') + request = self.factory.post(reverse('remove-ingests')) + response = RemoveCompletedIngestsView.as_view()(request) + self.assertEqual(response.status_code, 200, "Wrong HTTP code") def request_cleanup(self): print('*** Requesting cleanup ***') diff --git a/sip_assembly/views.py b/sip_assembly/views.py index e51b066..3910c62 100644 --- a/sip_assembly/views.py +++ b/sip_assembly/views.py @@ -89,6 +89,30 @@ def post(self, request): return Response({"detail": str(e)}, status=500) +class RemoveCompletedTransfersView(APIView): + """Removes completed transfers from Archivematica dashboard. Accepts POST requests only.""" + + def post(self, request): + log = logger.new(transaction_id=str(uuid4())) + try: + message = SIPActions().remove_completed_transfers() + return Response({"detail": message}, status=200) + except Exception as e: + return Response({"detail": str(e)}, status=500) + + +class RemoveCompletedIngestsView(APIView): + """Removes completed ingests from Archivematica dashboard. Accepts POST requests only.""" + + def post(self, request): + log = logger.new(transaction_id=str(uuid4())) + try: + message = SIPActions().remove_completed_ingests() + return Response({"detail": message}, status=200) + except Exception as e: + return Response({"detail": str(e)}, status=500) + + class CleanupRequestView(APIView): """Sends request to previous microservice to clean up source directory."""