Merge pull request digital-preservation#367 from digital-preservation…

…/manage-new-warc-365 Manage opening of WARC 1.1 and update identification of WARC 1.0
NEONScience · Jan 31, 2020 · b7d1dbc · b7d1dbc
2 parents e94e3bb + c9678e7
commit b7d1dbc
Show file tree

Hide file tree

Showing 8 changed files with 431 additions and 400 deletions.
diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/ResultPrinter.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/ResultPrinter.java
@@ -84,6 +84,8 @@ public class ResultPrinter {
     private static final String ARC_ARCHIVE = "x-fmt/219";
     private static final String ARC_ARCHIVE_OTHER = "fmt/410";
     private static final String WARC_ARCHIVE = "fmt/289";
+    private static final String WARC_ARCHIVE_1_1 = "fmt/1281";
+    private static final String WARC_ARCHIVE_1_0 = "fmt/1355";
     private static final String ISO_9660 = "fmt/468";
     private static final String SEVEN_ZIP = "fmt/484";
     private static final String BZIP2_ARCHIVE = "x-fmt/267";
@@ -227,6 +229,8 @@ public void print(final IdentificationResultCollection results,
                             }
                             break;
                         case WARC_ARCHIVE:
+                        case WARC_ARCHIVE_1_0:
+                        case WARC_ARCHIVE_1_1:
                             if ((archiveConfiguration.getExpandAllWebArchives() || containsCaseInsensitive("WARC", archiveConfiguration.getExpandWebArchiveTypes()))) {
                                 WarcArchiveContentIdentifier warcArchiveIdentifier =
                                         new WarcArchiveContentIdentifier(binarySignatureIdentifier,

diff --git a/...main/java/uk/gov/nationalarchives/droid/command/archive/WarcArchiveContentIdentifier.java b/...main/java/uk/gov/nationalarchives/droid/command/archive/WarcArchiveContentIdentifier.java
@@ -103,10 +103,11 @@ record = iterator.next();
                         && httpACCEPTED == record.getHttpHeader().statusCode) {
                         // no directory structure, so we use the full url as name
                         String name = record.header.warcTargetUriStr;
+                        Long time = record.header.warcDate == null ? null : record.header.warcDate.getTime();
 
                         RequestMetaData metaData = new RequestMetaData(
                             record.header.contentLength,
-                            record.header.warcDate.getTime(),
+                            time,
                             name);
 
                         final RequestIdentifier identifier = new RequestIdentifier(uri);

diff --git a/droid-command-line/src/main/resources/archive-puids.properties b/droid-command-line/src/main/resources/archive-puids.properties
@@ -36,7 +36,7 @@ archive.zip=x-fmt/263
 archive.tar=x-fmt/265
 archive.gz=x-fmt/266
 archive.arc=x-fmt/219, fmt/410
-archive.warc=fmt/289
+archive.warc=fmt/289, fmt/1281, fmt/1355
 archive.bz=x-fmt/267, x-fmt/268
 archive.7z=fmt/484
 archive.iso=fmt/468

diff --git a/droid-core-interfaces/src/main/resources/archive-puids.properties b/droid-core-interfaces/src/main/resources/archive-puids.properties
@@ -36,7 +36,7 @@ archive.zip=x-fmt/263
 archive.tar=x-fmt/265
 archive.gz=x-fmt/266
 archive.arc=x-fmt/219, fmt/410
-archive.warc=fmt/289
+archive.warc=fmt/289, fmt/1281, fmt/1355
 archive.bz=x-fmt/267, x-fmt/268
 archive.7z=fmt/484
 archive.iso=fmt/468

diff --git a/droid-parent/pom.xml b/droid-parent/pom.xml
@@ -52,6 +52,12 @@
         <developer>
             <name>Annabel Baynes</name>
         </developer>
+        <developer>
+            <name>Jeremie Charlet</name>
+        </developer>
+        <developer>
+            <name>Saurabh Parkhi</name>
+        </developer>
     </developers>
 
     <licenses>

diff --git a/droid-results/src/main/resources/META-INF/spring-results.xml b/droid-results/src/main/resources/META-INF/spring-results.xml
@@ -198,7 +198,7 @@ http://www.springframework.org/schema/context http://www.springframework.org/sch
                 <entry key="7Z" value="${archive.7z}"/>
                 <entry key="ISO" value="${archive.iso}"/>
                 <entry key="RAR" value="${archive.rar}"/>
-                <entry key="FAT" value="fmt/1087"/>
+                <entry key="FAT" value="${archive.fat}}"/>
             </map>
         </property>
     </bean>

diff --git a/droid-results/src/main/resources/archive-puids.properties b/droid-results/src/main/resources/archive-puids.properties
@@ -36,7 +36,7 @@ archive.zip=x-fmt/263
 archive.tar=x-fmt/265
 archive.gz=x-fmt/266
 archive.arc=x-fmt/219, fmt/410
-archive.warc=fmt/289
+archive.warc=fmt/289, fmt/1281, fmt/1355
 archive.bz=x-fmt/267, x-fmt/268
 archive.7z=fmt/484
 archive.iso=fmt/468