Merge pull request #343 from RockefellerArchiveCenter/issue-342

Adds integration tests for Archivematica
RockefellerArchiveCenter · Oct 29, 2024 · 717feeb · 717feeb
2 parents 450bccd + d0f3d26
commit 717feeb
Show file tree

Hide file tree

Showing 13 changed files with 225 additions and 4 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -53,4 +53,4 @@ jobs:
         run: pre-commit run --all-files --show-diff-on-failure
 
       - name: Run tests
-        run: docker compose exec -T $CONTAINER python manage.py test
+        run: docker compose exec -T $CONTAINER python manage.py test --exclude-tag=integration
diff --git a/README.md b/README.md
@@ -71,6 +71,11 @@ For an example of the data fornax expects to receive (both bags and JSON), see t
 |GET|/schema.json||200|Returns the OpenAPI schema for this application|
 
 
+## Archivematica Integration Testing
+When migrating Archivematica, it is necessary to test that Fornax can start transfers as expected. To run these integration tests, target the Python environment for this application and pass the `tag` flag to the tests management command: `env/bin/python manage.py test --tag=integration`.
+
+Running these tests will start a small package in all configured origins. This package will be set to not store the AIP or the DIP, but some manual cleanup will be required.
+
 ## Development
 This repository contains a configuration file for git [pre-commit](https://pre-commit.com/) hooks which help ensure that code is linted before it is checked into version control. It is strongly recommended that you install these hooks locally by installing pre-commit and running `pre-commit install`.
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   fornax-db:
     image: postgres:14.4

diff --git a/fixtures/integration/integration_test/bag-info.txt b/fixtures/integration/integration_test/bag-info.txt
@@ -0,0 +1,3 @@
+Bag-Software-Agent: bagit.py v1.8.1 <https://github.com/LibraryOfCongress/bagit-python>
+Bagging-Date: 2024-10-29
+Payload-Oxum: 20.1
diff --git a/fixtures/integration/integration_test/bagit.txt b/fixtures/integration/integration_test/bagit.txt
@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8
diff --git a/fixtures/integration/integration_test/data/objects/test.txt b/fixtures/integration/integration_test/data/objects/test.txt
@@ -0,0 +1 @@
+This is only a test!
diff --git a/fixtures/integration/integration_test/manifest-sha256.txt b/fixtures/integration/integration_test/manifest-sha256.txt
@@ -0,0 +1 @@
+51796743fccf4fbc83f1f9fcae1497916f43fa7e5a6b09a8ce91b41c3175534e  data/objects/test.txt
diff --git a/fixtures/integration/integration_test/manifest-sha512.txt b/fixtures/integration/integration_test/manifest-sha512.txt
@@ -0,0 +1 @@
+bc95a7991d3db6917e59e4d32a83e8abc38a86f5e52b25e92b472431dfeacd0d6f6c4dc40709a905c770983975e0829b0a85cc381059f61f7c2eb0ef83083d3a  data/objects/test.txt
diff --git a/fixtures/integration/integration_test/processingMCP.xml b/fixtures/integration/integration_test/processingMCP.xml
@@ -0,0 +1,139 @@
+<processingMCP>
+  <preconfiguredChoices>
+    <!-- Scan for viruses?: No -->
+    <preconfiguredChoice>
+      <appliesTo>856d2d65-cd25-49fa-8da9-cabb78292894</appliesTo>
+      <goToChain>63767e4b-9ce8-4fe2-8724-65cc1f763de0</goToChain>
+    </preconfiguredChoice>
+    <!-- Scan for viruses?: No -->
+    <preconfiguredChoice>
+      <appliesTo>1dad74a2-95df-4825-bbba-dca8b91d2371</appliesTo>
+      <goToChain>697c0883-798d-4af7-b8b6-101c7f709cd5</goToChain>
+    </preconfiguredChoice>
+    <!-- Scan for viruses?: No -->
+    <preconfiguredChoice>
+      <appliesTo>7e81f94e-6441-4430-a12d-76df09181b66</appliesTo>
+      <goToChain>77355172-b437-4324-9dcc-e2607ad27cb1</goToChain>
+    </preconfiguredChoice>
+    <!-- Scan for viruses?: No -->
+    <preconfiguredChoice>
+      <appliesTo>390d6507-5029-4dae-bcd4-ce7178c9b560</appliesTo>
+      <goToChain>63be6081-bee8-4cf5-a453-91893e31940f</goToChain>
+    </preconfiguredChoice>
+    <!-- Scan for viruses?: No -->
+    <preconfiguredChoice>
+      <appliesTo>97a5ddc0-d4e0-43ac-a571-9722405a0a9b</appliesTo>
+      <goToChain>7f5244fe-590b-4e38-beaf-0cf1ccb9e71b</goToChain>
+    </preconfiguredChoice>
+    <!-- Assign UUIDs to directories?: No -->
+    <preconfiguredChoice>
+      <appliesTo>bd899573-694e-4d33-8c9b-df0af802437d</appliesTo>
+      <goToChain>891f60d0-1ba8-48d3-b39e-dd0934635d29</goToChain>
+    </preconfiguredChoice>
+    <!-- Generate transfer structure report: No -->
+    <preconfiguredChoice>
+      <appliesTo>56eebd45-5600-4768-a8c2-ec0114555a3d</appliesTo>
+      <goToChain>e9eaef1e-c2e0-4e3b-b942-bfb537162795</goToChain>
+    </preconfiguredChoice>
+    <!-- Perform file format identification (Transfer): No -->
+    <preconfiguredChoice>
+      <appliesTo>f09847c2-ee51-429a-9478-a860477f6b8d</appliesTo>
+      <goToChain>1f77af0a-2f7a-468f-af8c-653a9e61ca4f</goToChain>
+    </preconfiguredChoice>
+    <!-- Extract packages?: No -->
+    <preconfiguredChoice>
+      <appliesTo>dec97e3c-5598-4b99-b26e-f87a435a6b7f</appliesTo>
+      <goToChain>79f1f5af-7694-48a4-b645-e42790bbf870</goToChain>
+    </preconfiguredChoice>
+    <!-- Perform policy checks on originals?: No -->
+    <preconfiguredChoice>
+      <appliesTo>70fc7040-d4fb-4d19-a0e6-792387ca1006</appliesTo>
+      <goToChain>3e891cc4-39d2-4989-a001-5107a009a223</goToChain>
+    </preconfiguredChoice>
+    <!-- Examine contents?: Skip examine contents -->
+    <preconfiguredChoice>
+      <appliesTo>accea2bf-ba74-4a3a-bb97-614775c74459</appliesTo>
+      <goToChain>e0a39199-c62a-4a2f-98de-e9d1116460a8</goToChain>
+    </preconfiguredChoice>
+    <!-- Create SIP(s): Create single SIP and continue processing -->
+    <preconfiguredChoice>
+      <appliesTo>bb194013-597c-4e4a-8493-b36d190f8717</appliesTo>
+      <goToChain>61cfa825-120e-4b17-83e6-51a42b67d969</goToChain>
+    </preconfiguredChoice>
+    <!-- Perform file format identification (Ingest): No, use existing data -->
+    <preconfiguredChoice>
+      <appliesTo>7a024896-c4f7-4808-a240-44c87c762bc5</appliesTo>
+      <goToChain>3c1faec7-7e1e-4cdd-b3bd-e2f05f4baa9b</goToChain>
+    </preconfiguredChoice>
+    <!-- Normalize: Do not normalize -->
+    <preconfiguredChoice>
+      <appliesTo>cb8e5706-e73f-472f-ad9b-d1236af8095f</appliesTo>
+      <goToChain>89cb80dd-0636-464f-930d-57b61e3928b2</goToChain>
+    </preconfiguredChoice>
+    <!-- Normalize: Do not normalize -->
+    <preconfiguredChoice>
+      <appliesTo>7509e7dc-1e1b-4dce-8d21-e130515fce73</appliesTo>
+      <goToChain>e8544c5e-9cbb-4b8f-a68b-6d9b4d7f7362</goToChain>
+    </preconfiguredChoice>
+    <!-- Approve normalization: Yes -->
+    <preconfiguredChoice>
+      <appliesTo>de909a42-c5b5-46e1-9985-c031b50e9d30</appliesTo>
+      <goToChain>1e0df175-d56d-450d-8bee-7df1dc7ae815</goToChain>
+    </preconfiguredChoice>
+    <!-- Choose thumbnail mode: No -->
+    <preconfiguredChoice>
+      <appliesTo>498f7a6d-1b8c-431a-aa5d-83f14f3c5e65</appliesTo>
+      <goToChain>972fce6c-52c8-4c00-99b9-d6814e377974</goToChain>
+    </preconfiguredChoice>
+    <!-- Perform policy checks on preservation derivatives?: No -->
+    <preconfiguredChoice>
+      <appliesTo>153c5f41-3cfb-47ba-9150-2dd44ebc27df</appliesTo>
+      <goToChain>b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546</goToChain>
+    </preconfiguredChoice>
+    <!-- Perform policy checks on access derivatives?: No -->
+    <preconfiguredChoice>
+      <appliesTo>8ce07e94-6130-4987-96f0-2399ad45c5c2</appliesTo>
+      <goToChain>76befd52-14c3-44f9-838f-15a4e01624b0</goToChain>
+    </preconfiguredChoice>
+    <!-- Bind PIDs?: No -->
+    <preconfiguredChoice>
+      <appliesTo>a2ba5278-459a-4638-92d9-38eb1588717d</appliesTo>
+      <goToChain>44a7c397-8187-4fd2-b8f7-c61737c4df49</goToChain>
+    </preconfiguredChoice>
+    <!-- Document empty directories?: No -->
+    <preconfiguredChoice>
+      <appliesTo>d0dfa5fc-e3c2-4638-9eda-f96eea1070e0</appliesTo>
+      <goToChain>65273f18-5b4e-4944-af4f-09be175a88e8</goToChain>
+    </preconfiguredChoice>
+    <!-- Reminder: add metadata if desired: Continue -->
+    <preconfiguredChoice>
+      <appliesTo>eeb23509-57e2-4529-8857-9d62525db048</appliesTo>
+      <goToChain>5727faac-88af-40e8-8c10-268644b0142d</goToChain>
+    </preconfiguredChoice>
+    <!-- Transcribe SIP contents?: No -->
+    <preconfiguredChoice>
+      <appliesTo>82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6</appliesTo>
+      <goToChain>0a24787c-00e3-4710-b324-90e792bfb484</goToChain>
+    </preconfiguredChoice>
+    <!-- Perform file format identification (Submission documentation & metadata): No -->
+    <preconfiguredChoice>
+      <appliesTo>087d27be-c719-47d8-9bbb-9a7d8b609c44</appliesTo>
+      <goToChain>782bbf56-e220-48b5-9eb6-6610583f2072</goToChain>
+    </preconfiguredChoice>
+    <!-- Select compression algorithm: Uncompressed -->
+    <preconfiguredChoice>
+      <appliesTo>01d64f58-8295-4b7b-9cab-8f1b153a504f</appliesTo>
+      <goToChain>dc04c4c0-07ea-4796-b643-66d967ed33a4</goToChain>
+    </preconfiguredChoice>
+    <!-- Upload DIP: Do not upload DIP -->
+    <preconfiguredChoice>
+      <appliesTo>92879a29-45bf-4f0b-ac43-e64474f0f2f9</appliesTo>
+      <goToChain>6eb8ebe7-fab3-4e4c-b9d7-14de17625baa</goToChain>
+    </preconfiguredChoice>
+    <!-- Store DIP?: Reject DIP -->
+    <preconfiguredChoice>
+      <appliesTo>5e58066d-e113-4383-b20b-f301ed4d751c</appliesTo>
+      <goToChain>4500f34e-f004-4ccf-8720-5c38d0be2254</goToChain>
+    </preconfiguredChoice>
+  </preconfiguredChoices>
+</processingMCP>
diff --git a/fixtures/integration/integration_test/tagmanifest-sha256.txt b/fixtures/integration/integration_test/tagmanifest-sha256.txt
@@ -0,0 +1,5 @@
+e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
+b4bf4840f19c7dc4faeed7fb497d843bff3debd8833d48cc008024762a7570f9 bag-info.txt
+0ad557aec3b83e1f5335d8a505707be6cef0ee34077e2857bc76f49c644c5f88 manifest-sha512.txt
+2f713d55543ededbde94d8c9d318f4205049641ad12764e1c04dd5d84f8925f3 manifest-sha256.txt
+a84ae74642dc2ce9df01ef962b62be8527b8fd8a8286228998a694c2b1d1c3b1 processingMCP.xml
diff --git a/fixtures/integration/integration_test/tagmanifest-sha512.txt b/fixtures/integration/integration_test/tagmanifest-sha512.txt
@@ -0,0 +1,5 @@
+418dcfbe17d5f4b454b18630be795462cf7da4ceb6313afa49451aa2568e41f7ca3d34cf0280c7d056dc5681a70c37586aa1755620520b9198eede905ba2d0f6 bagit.txt
+5b8f8dcc9809a7c3e1a1bad43e8d663a2c7cd2559daf782c078243b0a11dd9f2c19e67bb65dd08ac4efb2401c20be16d8887626c74b5cc63e362c2a033b97630 bag-info.txt
+205d3219e132de2e2b6f387b11807611f7dfdc1d3e2020af7bba24ffba94f404ae2e9c73f8c09cef35e2afa4481225caf5f656561252f519b748d7f9b32c115a manifest-sha512.txt
+e67660c6a434bb0e316957bb79f30370c8dd4d5a61d1a63c9fc34cf87dd0f700f918a7962ce4c7bd3f3771f0fe14e1d1bdc52e61c7c4fac597c1299ad593e615 manifest-sha256.txt
+3b7cb8f5f782d2ba694ef0ab493debcb7fd9591511bcf921bdef5e3a856f6dfbd9276fd034b8738665951506b776140ae9c3d5e7a0f742f24f9092755953da9f processingMCP.xml
diff --git a/fixtures/integration/valid_rights.csv b/fixtures/integration/valid_rights.csv
@@ -0,0 +1,3 @@
+file,basis,status,determination_date,jurisdiction,start_date,end_date,terms,citation,note,grant_act,grant_restriction,grant_start_date,grant_end_date,grant_note,doc_id_type,doc_id_value,doc_id_role
+data/objects/51538919827_742b48272f_q.jpg,other,,,,2021-01-01,,,,These materials have an online embargo period of 50 years.,publish,disallow,1955-01-01,2006-12-31,,,,
+data/objects/metadata/submissionDocumentation/documentation.txt,other,,,,2021-01-01,,,,These materials have an online embargo period of 50 years.,publish,disallow,1955-01-01,2006-12-31,,,,
diff --git a/sip_assembly/tests.py b/sip_assembly/tests.py
@@ -8,7 +8,8 @@
 
 import bagit
 from amclient import errors, utils
-from django.test import TestCase
+from asterism import bagit_helpers, file_helpers
+from django.test import TestCase, tag
 from django.urls import reverse
 
 from fornax import settings
@@ -26,6 +27,7 @@
 bag_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'bags')
 csv_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'csv_creation')
 processing_config_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'processing_configs')
+integration_fixture_dir = join(settings.BASE_DIR, 'fixtures', 'integration')
 
 
 class CsvCreatorTests(TestCase):
@@ -359,3 +361,59 @@ def test_request_cleanup_view(self, mock_request):
     def test_health_check_view(self):
         """Tests the health check view."""
         self.assert_status_code("get", reverse("ping"), 200)
+
+
+@tag("integration")
+class ArchivematicaIntegrationTests(TestCase):
+    """Provides integration tests for use during Archivematica migration.
+
+    Running these tests will start a small transfer in each configured origin.
+    This transfer is set to not store either an AIP or a DIP.
+    """
+
+    def test_processing_config(self):
+        """Ensure a processing configuration is returned as expected."""
+        for origin in settings.ARCHIVEMATICA_ORIGINS:
+            client = ArchivematicaClientMixin().get_client(origin)
+            config = ArchivematicaClientMixin().get_processing_config(client)
+            self.assertIsInstance(config, str)
+
+    def test_close_packages(self):
+        """Ensure packages are closed as expected."""
+        for type in ['transfer', 'ingest']:
+            removed = ArchivematicaClientMixin().remove_completed(type)
+            self.assertIsInstance(removed, tuple)
+            self.assertIn(type, removed[0])
+
+    def test_rights_validation(self):
+        """Ensure rights.csv files are validated as expected."""
+        with open(join(integration_fixture_dir, 'valid_rights.csv'), "r") as rightscsv:
+            for origin in settings.ARCHIVEMATICA_ORIGINS:
+                client = ArchivematicaClientMixin().get_client(origin)
+                client.enhanced_errors = True
+                valid = client.validate_csv("rights", rightscsv)
+                self.assertEqual(valid, {'valid': True})
+                rightscsv.seek(0)
+
+    def test_start_package(self):
+        """Ensures packages are started as expected.
+
+        This test will start a small package named `integration_test` in each configured origin.
+        """
+        bag_name = 'integration_test'
+        transfer_path = join(integration_fixture_dir, bag_name)
+        for origin in settings.ARCHIVEMATICA_ORIGINS:
+            client = ArchivematicaClientMixin().get_client(origin)
+            client.processing_config = 'integration_test'
+            config = ArchivematicaClientMixin().get_processing_config(client)
+            with open(join(transfer_path, 'processingMCP.xml'), 'w') as f:
+                f.write(config)
+            bagit_helpers.update_manifests(transfer_path)
+            tar_path = join(settings.DEST_DIR, f"{bag_name}.tar.gz")
+            file_helpers.make_tarfile(transfer_path, tar_path, compressed=True)
+
+            client.transfer_directory = f"{bag_name}.tar.gz"
+            client.transfer_name = bag_name
+            client.transfer_type = 'zipped bag'
+            started = client.create_package()
+            self.assertIsInstance(started, dict)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		BagIt-Version: 0.97
		Tag-File-Character-Encoding: UTF-8
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		51796743fccf4fbc83f1f9fcae1497916f43fa7e5a6b09a8ce91b41c3175534e data/objects/test.txt
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		bc95a7991d3db6917e59e4d32a83e8abc38a86f5e52b25e92b472431dfeacd0d6f6c4dc40709a905c770983975e0829b0a85cc381059f61f7c2eb0ef83083d3a data/objects/test.txt