Merge pull request #142 from RockefellerArchiveCenter/v1.1

v1.1

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+ignore = E501
+exclude = .git,__pycache__,sip_assembly/migrations

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+- repo: https://github.com/pre-commit/mirrors-autopep8
+  rev: v1.5
+  hooks:
+  - id: autopep8
+    args:
+      - --in-place
+      - --aggressive
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 3.7.9
+  hooks:
+  - id: flake8
+- repo: git://github.com/doublify/pre-commit-isort
+  rev: v4.3.0
+  hooks:
+  - id: isort

.travis.yml

@@ -13,13 +13,15 @@ before_install:
   - cp fornax/config.py.example fornax/config.py
   - docker-compose up -d
   - sleep 20s
-  - docker-compose exec fornax-web pip install coverage
+  - docker-compose exec fornax-web pip install coverage pre-commit
+  - docker-compose exec fornax-web pre-commit install
 install: true
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
 script:
+  - docker-compose exec fornax-web pre-commit run --all-files --show-diff-on-failure
   - docker-compose exec fornax-web coverage run manage.py test
 after_script:
   - docker-compose exec fornax-web coverage xml

README.md

@@ -54,8 +54,6 @@ fornax has six services, all of which are exposed via HTTP endpoints (see [Route
 * Cleanup - removes files from the destination directory.
 * Request Cleanup - sends a POST request to another service requesting cleanup of the source directory. fornax only has read access for this directory.
 
-  ![SIP Assembly diagram](fornax-services.png)
-
 For an example of the data fornax expects to receive (both bags and JSON), see the `fixtures/` directory
 
 

docker-compose.yml

@@ -10,7 +10,6 @@ services:
     entrypoint: /code/entrypoint.sh
     volumes:
       - .:/code
-      - ~/.am/ss-location-data:/code/archivematica_transfer_source/
     ports:
       - "8003:8003"
     depends_on:

entrypoint.sh

@@ -14,8 +14,5 @@ fi
 echo "Apply database migrations"
 python manage.py migrate
 
-echo "Create dirs"
-python manage.py shell < make_dirs.py
-
 echo "Starting server"
 python manage.py runserver 0.0.0.0:8003

fornax/config.py.example

@@ -15,11 +15,8 @@ ALLOWED_HOSTS = ['fornax-web', 'localhost']
 BASE_DIR = ''
 
 SRC_DIR = '{}/src'.format(BASE_DIR)
-TEST_SRC_DIR = '{}/src_test'.format(BASE_DIR)
 TMP_DIR = '{}/tmp'.format(BASE_DIR)
-TEST_TMP_DIR = '{}/tmp_test'.format(BASE_DIR)
 DEST_DIR = '{}/dest'.format(BASE_DIR)
-TEST_DEST_DIR = '{}/dest_test'.format(BASE_DIR)
 
 PROCESSING_CONFIG_DIR = 'processing_configs'
 PROCESSING_CONFIG = 'processingMCP.xml'
@@ -51,4 +48,6 @@ ARCHIVEMATICA = {
   }
 }
 
-STATIC_ROOT = '/static'
+STATIC_ROOT = "/static"
+
+CLEANUP_URL = "http://ursa-major-web:8005/cleanup/"

fornax/settings.py

@@ -11,6 +11,7 @@
 """
 
 import os
+
 from fornax import config as CF
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -40,6 +41,7 @@
     'sip_assembly',
     'rest_framework',
     'health_check',
+    'asterism',
 ]
 
 MIDDLEWARE = [
@@ -115,14 +117,12 @@
 STATIC_ROOT = CF.STATIC_ROOT
 
 SRC_DIR = CF.SRC_DIR
-TEST_SRC_DIR = CF.TEST_SRC_DIR
 TMP_DIR = CF.TMP_DIR
-TEST_TMP_DIR = CF.TEST_TMP_DIR
 DEST_DIR = CF.DEST_DIR
-TEST_DEST_DIR = CF.TEST_DEST_DIR
 PROCESSING_CONFIG_DIR = CF.PROCESSING_CONFIG_DIR
 PROCESSING_CONFIG = CF.PROCESSING_CONFIG
 ARCHIVEMATICA = CF.ARCHIVEMATICA
+CLEANUP_URL = CF.CLEANUP_URL
 
 REST_FRAMEWORK = {
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',

fornax/urls.py

@@ -15,16 +15,12 @@
 """
 from django.conf.urls import url
 from django.urls import include
-from sip_assembly.views import (
-    SIPViewSet,
-    SIPAssemblyView,
-    CreatePackageView,
-    RemoveCompletedTransfersView,
-    RemoveCompletedIngestsView,
-    CleanupRoutineView,
-    CleanupRequestView)
 from rest_framework import routers
 from rest_framework.schemas import get_schema_view
+from sip_assembly.views import (CleanupRequestView, CleanupRoutineView,
+                                CreatePackageView, RemoveCompletedIngestsView,
+                                RemoveCompletedTransfersView, SIPAssemblyView,
+                                SIPViewSet)
 
 router = routers.DefaultRouter()
 router.register(r'sips', SIPViewSet)

requirements.txt

@@ -1,14 +1,14 @@
 amclient==1.0.0
+asterism==0.5.2
 bagit==1.7.0
 certifi==2019.11.28
 chardet==3.0.4
 click==7.0
 clinner==1.12.3
 colorlog==4.0.2
 csvvalidator==1.2
-Django==2.2.8
+Django==2.2.10
 djangorestframework==3.10.3
-git+https://github.com/RockefellerArchiveCenter/[email protected]#egg=asterism
 gitdb2==2.0.6
 GitPython==3.0.5
 health-check==3.4.1

sip_assembly/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

sip_assembly/migrations/0001_initial.py

@@ -15,13 +15,34 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SIP',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('process_status', models.CharField(choices=[(10, 'New SIP created'), (20, 'SIP files moved to processing'), (30, 'SIP validated according to BagIt'), (30, 'SIP restructured'), (40, 'PREMIS CSV rights added to SIP'), (50, 'Submission documentation added to SIP'), (60, 'SIP bag-info.txt updated'), (70, 'SIP Manifests updated'), (90, 'SIP Delivered to Archivematica Transfer Source')], max_length=100)),
+                ('id', models.AutoField(auto_created=True,
+                                        primary_key=True, serialize=False, verbose_name='ID')),
+                ('process_status',
+                 models.CharField(choices=[(10,
+                                            'New SIP created'),
+                                           (20,
+                                            'SIP files moved to processing'),
+                                           (30,
+                                            'SIP validated according to BagIt'),
+                                           (30,
+                                            'SIP restructured'),
+                                           (40,
+                                            'PREMIS CSV rights added to SIP'),
+                                           (50,
+                                            'Submission documentation added to SIP'),
+                                           (60,
+                                            'SIP bag-info.txt updated'),
+                                           (70,
+                                            'SIP Manifests updated'),
+                                           (90,
+                                            'SIP Delivered to Archivematica Transfer Source')],
+                                  max_length=100)),
                 ('bag_path', models.CharField(max_length=100)),
                 ('bag_identifier', models.CharField(max_length=255, unique=True)),
                 ('created', models.DateTimeField(auto_now=True)),
                 ('last_modified', models.DateTimeField(auto_now_add=True)),
-                ('data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
+                ('data', django.contrib.postgres.fields.jsonb.JSONField(
+                    blank=True, null=True)),
             ],
         ),
     ]

sip_assembly/migrations/0002_auto_20180910_1744.py

@@ -13,6 +13,28 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='sip',
             name='process_status',
-            field=models.CharField(choices=[(10, 'New SIP created'), (20, 'SIP files moved to processing'), (30, 'SIP validated according to BagIt'), (30, 'SIP restructured'), (40, 'PREMIS CSV rights added to SIP'), (50, 'Submission documentation added to SIP'), (60, 'SIP bag-info.txt updated'), (70, 'Archivematica processing config added'), (80, 'SIP Manifests updated'), (90, 'SIP Delivered to Archivematica Transfer Source')], max_length=100),
+            field=models.CharField(
+                choices=[
+                    (10,
+                     'New SIP created'),
+                    (20,
+                     'SIP files moved to processing'),
+                    (30,
+                     'SIP validated according to BagIt'),
+                    (30,
+                     'SIP restructured'),
+                    (40,
+                     'PREMIS CSV rights added to SIP'),
+                    (50,
+                     'Submission documentation added to SIP'),
+                    (60,
+                     'SIP bag-info.txt updated'),
+                    (70,
+                     'Archivematica processing config added'),
+                    (80,
+                     'SIP Manifests updated'),
+                    (90,
+                     'SIP Delivered to Archivematica Transfer Source')],
+                max_length=100),
         ),
     ]

sip_assembly/migrations/0003_auto_20181112_2040.py

@@ -13,6 +13,16 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='sip',
             name='process_status',
-            field=models.CharField(choices=[(10, 'New SIP created'), (20, 'SIP assembled and delivered to Archivematica'), (30, 'SIP started in Archivematica'), (40, 'SIP approved in Archivematica')], max_length=100),
+            field=models.CharField(
+                choices=[
+                    (10,
+                     'New SIP created'),
+                    (20,
+                     'SIP assembled and delivered to Archivematica'),
+                    (30,
+                     'SIP started in Archivematica'),
+                    (40,
+                     'SIP approved in Archivematica')],
+                max_length=100),
         ),
     ]

sip_assembly/migrations/0004_auto_20181118_2151.py

@@ -13,6 +13,18 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='sip',
             name='process_status',
-            field=models.CharField(choices=[(10, 'New SIP created'), (20, 'SIP assembled and delivered to Archivematica'), (30, 'SIP started in Archivematica'), (40, 'SIP approved in Archivematica'), (50, 'SIP removed from src directory')], max_length=100),
+            field=models.CharField(
+                choices=[
+                    (10,
+                     'New SIP created'),
+                    (20,
+                     'SIP assembled and delivered to Archivematica'),
+                    (30,
+                     'SIP started in Archivematica'),
+                    (40,
+                     'SIP approved in Archivematica'),
+                    (50,
+                     'SIP removed from src directory')],
+                max_length=100),
         ),
     ]

sip_assembly/migrations/0005_sip_origin.py

@@ -13,6 +13,15 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='sip',
             name='origin',
-            field=models.CharField(choices=[('aurora', 'Aurora'), ('legacy_digital', 'Legacy Digital Processing'), ('digitization', 'Digitization')], default='aurora', max_length=20),
+            field=models.CharField(
+                choices=[
+                    ('aurora',
+                     'Aurora'),
+                    ('legacy_digital',
+                     'Legacy Digital Processing'),
+                    ('digitization',
+                     'Digitization')],
+                default='aurora',
+                max_length=20),
         ),
     ]

sip_assembly/migrations/0006_auto_20200317_0220.py

@@ -0,0 +1,38 @@
+# Generated by Django 2.2.10 on 2020-03-17 02:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sip_assembly', '0005_sip_origin'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sip',
+            name='type',
+            field=models.CharField(blank=True, choices=[('aip', 'Archival Information Package'), ('dip', 'Dissemination Information Package')], max_length=50, null=True),
+        ),
+        migrations.AlterField(
+            model_name='sip',
+            name='bag_path',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='sip',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True),
+        ),
+        migrations.AlterField(
+            model_name='sip',
+            name='last_modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AlterField(
+            model_name='sip',
+            name='process_status',
+            field=models.IntegerField(),
+        ),
+    ]

sip_assembly/models.py

@@ -1,8 +1,7 @@
-from django.contrib.postgres.fields import JSONField
-from django.db import models
+from asterism.models import BasePackage
 
 
-class SIP(models.Model):
+class SIP(BasePackage):
     CREATED = 10
     ASSEMBLED = 20
     STARTED = 30
@@ -15,15 +14,3 @@ class SIP(models.Model):
         (APPROVED, "SIP approved in Archivematica"),
         (CLEANED_UP, "SIP removed from src directory")
     )
-    process_status = models.CharField(max_length=100, choices=PROCESS_STATUS_CHOICES)
-    bag_path = models.CharField(max_length=100)
-    bag_identifier = models.CharField(max_length=255, unique=True)
-    created = models.DateTimeField(auto_now=True)
-    last_modified = models.DateTimeField(auto_now_add=True)
-    data = JSONField(null=True, blank=True)
-    ORIGIN_CHOICES = (
-        ('aurora', 'Aurora'),
-        ('legacy_digital', 'Legacy Digital Processing'),
-        ('digitization', 'Digitization')
-    )
-    origin = models.CharField(max_length=20, choices=ORIGIN_CHOICES, default='aurora')