S3

README.md

@@ -1,3 +1,25 @@
+
+# IMPORTANT NOTE
+This is an experimental fork of Photon implementing extensions to validate IMF App#5 ACES packages.
+These extensions are experimental, work-in-progress and not intended for production.
+
+You can find the Original Photon software [here](https://github.com/netflix/photon).
+
+# How to use
+The IMPAnalyzer now supports the following option:
+```
+java -cp ./build/libs/*: com.netflix.imflibrary.app.IMPAnalyzer <directory>
+```
+```
+options:            
+-a, --application APPSTRING      IMF Application to test against, supported values for APPSTRING are app2or2E, app5 or all (default: app2or2E)
+```
+
+Limitation:
+The option "-a" is currently supported for the ```<package_directory>``` argument only.
+
+Please note, by default Photon tests against the App #2/#2E specification.
+
 # Photon
 
 Photon is a Java implementation of the Interoperable Master Format (IMF) standard. IMF is a SMPTE standard whose core constraints are defined in the specification st2067-2:2013. Photon offers tools for parsing, interpreting and validating constituent files that make an Interoperable Master Package (IMP). These include AssetMap (st429-9:2014), PackingList (st429-8:2007), Composition Playlist (st2067-3:2013), and the essence containing IMF track file (st2067-5:2013) which follows the Material eXchange Format (MXF) format (st377-1:2011). Specifically, Photon parses and completely reads an MXF file containing a single audio or video essence as defined by the IMF Essence Component specification (st2067-5:2013) and serializes the metadata into the IMF Composition Playlist structure.

build.gradle

@@ -2,6 +2,9 @@ buildscript {
     repositories {
         jcenter()
     }
+    dependencies {
+        classpath "io.spring.gradle:dependency-management-plugin:1.0.3.RELEASE"
+    }
 }
 
 plugins {
@@ -11,6 +14,8 @@ plugins {
     id 'findbugs'
 }
 
+apply plugin: "io.spring.dependency-management"
+
 group = 'com.netflix.photon'
 
 sourceCompatibility = 1.8
@@ -55,14 +60,20 @@ javadoc {
     exclude "**/org/w3/_2000/_09/xmldsig_/**"
 }
 
+dependencyManagement {
+    imports {
+        mavenBom 'com.amazonaws:aws-java-sdk-bom:1.11.228'
+    }
+}
+
 dependencies {
     compile 'com.google.code.findbugs:jsr305:3.0.1'
     compile "org.slf4j:slf4j-api:1.7+"
     compile "org.slf4j:slf4j-log4j12:1.7+"
     /**
      * Following includes the RegXMLLib dependency from Maven Central.
      */
-    compile "com.sandflow:regxmllib:1.1.1"
+    compile "com.sandflow:regxmllib:1.1.2"
     /**
      * Following should be enabled and the above should be disabled
      * when necessary to verify changes to the RegXMLLib library that are
@@ -71,6 +82,8 @@ dependencies {
     /*compile "com.sandflow:regxmllib:${revRegXMLSNAPSHOT}"*/
     testCompile "org.mockito:mockito-all:1.9+"
     testCompile "org.testng:testng:6.+"
+    compile 'com.amazonaws:aws-java-sdk-s3'
+    testCompile group: 'junit', name: 'junit', version: '4.11'
 }
 
 test {
@@ -99,4 +112,4 @@ task wrapper(type: Wrapper) {
 
 //Following line will enable including compile time dependencies as a part of the
 //compile and build process.
-//assemble.dependsOn getDependencies
+//assemble.dependsOn getDependencies

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Nov 24 11:41:36 PST 2015
+#Thu Aug 23 08:40:04 EEST 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.9-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.9-all.zip

src/main/java/com/netflix/imflibrary/Colorimetry.java

@@ -21,6 +21,7 @@ public enum Colorimetry {
     Color5(ColorPrimaries.ITU2020,    TransferCharacteristic.ITU2020,           CodingEquation.ITU2020NCL),
     Color6(ColorPrimaries.P3D65,      TransferCharacteristic.SMPTEST2084,       CodingEquation.None),
     Color7(ColorPrimaries.ITU2020,    TransferCharacteristic.SMPTEST2084,       CodingEquation.ITU2020NCL),
+    Color_App5_AP0(ColorPrimaries.ACES,    TransferCharacteristic.Linear,       CodingEquation.None),
     Unknown(ColorPrimaries.Unknown,   TransferCharacteristic.Unknown,           CodingEquation.Unknown);
 
 
@@ -85,6 +86,7 @@ public static enum TransferCharacteristic {
         IEC6196624xvYCC(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0E.2B.34.04.01.01.0D.04.01.01.01.01.08.00.00")),
         ITU2020(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0E.2B.34.04.01.01.0E.04.01.01.01.01.09.00.00")),
         SMPTEST2084(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0E.2B.34.04.01.01.0D.04.01.01.01.01.0A.00.00")),
+        Linear(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0e.2b.34.04.01.01.06.04.01.01.01.01.06.00.00")),
         Unknown(null);
 
         private final UL transferCharacteristicUL;
@@ -112,6 +114,7 @@ public static enum ColorPrimaries {
         ITU709(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0E.2B.34.04.01.01.06.04.01.01.01.03.03.00.00")),
         ITU2020(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0E.2B.34.04.01.01.0D.04.01.01.01.03.04.00.00")),
         P3D65(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0E.2B.34.04.01.01.0D.04.01.01.01.03.06.00.00")),
+        ACES(UL.fromULAsURNStringToUL("urn:smpte:ul:06.0e.2b.34.04.01.01.0d.04.01.01.01.03.07.00.00")),
         Unknown(null);
 
         private final UL colorPrimariesUL;

src/main/java/com/netflix/imflibrary/IMPDelivery.java

@@ -22,14 +22,14 @@
 import com.netflix.imflibrary.st0429_9.AssetMap;
 import com.netflix.imflibrary.st0429_9.BasicMapProfileV2FileSet;
 import com.netflix.imflibrary.st0429_9.BasicMapProfileV2MappedFileSet;
+import com.netflix.imflibrary.utils.FileLocator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
 import javax.annotation.concurrent.Immutable;
 import javax.xml.bind.JAXBException;
 import javax.xml.parsers.ParserConfigurationException;
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -64,7 +64,7 @@ public IMPDelivery(BasicMapProfileV2FileSet basicMapProfileV2FileSet) throws IOE
         for (AssetMap.Asset packingListAsset : packingListAssets)
         {
             URI absolutePackingListURI = basicMapProfileV2FileSet.getAbsoluteAssetMapURI().resolve(packingListAsset.getPath());
-            PackingList packingList = new PackingList(new File(absolutePackingListURI));
+            PackingList packingList = new PackingList(FileLocator.fromLocation(absolutePackingListURI));
 
             List<IMPAsset> referencedAssets = new ArrayList<>();
             for (PackingList.Asset referencedAsset : packingList.getAssets())
@@ -117,9 +117,9 @@ public String toString()
 
     public static void main(String[] args) throws ParserConfigurationException, SAXException, IOException, URISyntaxException, JAXBException
     {
-        File rootFile = new File(args[0]);
+        FileLocator rootFileLocator = FileLocator.fromLocation(args[0]);
 
-        BasicMapProfileV2MappedFileSet basicMapProfileV2MappedFileSet = new BasicMapProfileV2MappedFileSet(rootFile);
+        BasicMapProfileV2MappedFileSet basicMapProfileV2MappedFileSet = new BasicMapProfileV2MappedFileSet(rootFileLocator);
         BasicMapProfileV2FileSet basicMapProfileV2FileSet = new BasicMapProfileV2FileSet(basicMapProfileV2MappedFileSet);
         IMPDelivery impDelivery = new IMPDelivery(basicMapProfileV2FileSet);
 

src/main/java/com/netflix/imflibrary/app/IMFTrackFileCPLBuilder.java

@@ -22,6 +22,9 @@
 import com.netflix.imflibrary.IMFErrorLoggerImpl;
 import com.netflix.imflibrary.st0377.HeaderPartition;
 import com.netflix.imflibrary.utils.ErrorLogger;
+import com.netflix.imflibrary.utils.FileLocator;
+import com.netflix.imflibrary.utils.RegXMLLibHelper;
+import com.netflix.imflibrary.utils.ResourceByteRangeProvider;
 import com.netflix.imflibrary.writerTools.CompositionPlaylistBuilder_2013;
 import com.sandflow.smpte.klv.Triplet;
 import com.netflix.imflibrary.KLVPacket;
@@ -40,10 +43,7 @@
 import org.smpte_ra.schemas.st2067_2_2013.SequenceType;
 import org.smpte_ra.schemas.st2067_2_2013.UserTextType;
 import com.netflix.imflibrary.st0377.header.InterchangeObject;
-import com.netflix.imflibrary.utils.FileByteRangeProvider;
-import com.netflix.imflibrary.utils.ResourceByteRangeProvider;
 import com.netflix.imflibrary.writerTools.IMFCPLObjectFieldsFactory;
-import com.netflix.imflibrary.utils.RegXMLLibHelper;
 import com.netflix.imflibrary.writerTools.utils.IMFUUIDGenerator;
 import com.netflix.imflibrary.writerTools.utils.IMFUtils;
 import org.slf4j.Logger;
@@ -83,27 +83,27 @@ final class IMFTrackFileCPLBuilder {
     private final RegXMLLibHelper regXMLLibHelper;
     private final File workingDirectory;
     private final org.smpte_ra.schemas.st2067_2_2013.CompositionPlaylistType cplRoot;
-    private final File mxfFile;
+    private final FileLocator mxfFile;
     private final String fileName;
 
 
     /**
      * A constructor for the IMFTrackFileCPLBuilder class. This class creates an IMF CPL representation of an IMF Essence
      * @param workingDirectory - A location on a file system used for processing the essence.
      *                         This would also be the location where the CPL representation of the IMFEssence would be written into.
-     * @param essenceFile - File representing an IMF Essence
+     * @param essenceFileLocator - FileLocator representing an IMF Essence
      * @throws IOException - any I/O related error will be exposed through an IOException
      */
-    public IMFTrackFileCPLBuilder(File workingDirectory, File essenceFile) throws IOException {
-        ResourceByteRangeProvider resourceByteRangeProvider = new FileByteRangeProvider(essenceFile);
+    public IMFTrackFileCPLBuilder(File workingDirectory, FileLocator essenceFileLocator) throws IOException {
+        ResourceByteRangeProvider resourceByteRangeProvider = essenceFileLocator.getResourceByteRangeProvider();
         this.imfTrackFileReader = new IMFTrackFileReader(workingDirectory, resourceByteRangeProvider);
         IMFErrorLogger imfErrorLogger = new IMFErrorLoggerImpl();
         KLVPacket.Header primerPackHeader = this.imfTrackFileReader.getPrimerPackHeader(imfErrorLogger);
         this.regXMLLibHelper = new RegXMLLibHelper(primerPackHeader, this.imfTrackFileReader.getByteProvider(primerPackHeader));
         this.workingDirectory = workingDirectory;
         /*Peek into the CompositionPlayListType and recursively construct its constituent fields*/
         this.cplRoot = IMFCPLObjectFieldsFactory.constructCompositionPlaylistType_2013();
-        this.mxfFile = essenceFile;
+        this.mxfFile = essenceFileLocator;
         this.fileName = this.mxfFile.getName();
     }
 
@@ -419,30 +419,30 @@ private static String usage()
         return sb.toString();
     }
 
-    public static void main(String[] args){
+    public static void main(String[] args) throws IOException{
 
         if (args.length != 2)
         {
             logger.error(usage());
             throw new IllegalArgumentException("Invalid parameters");
         }
 
-        File inputFile = new File(args[0]);
-        if(!inputFile.exists()){
-            logger.error(String.format("File %s does not exist", inputFile.getAbsolutePath()));
+        FileLocator inputFileLocator = FileLocator.fromLocation(args[0]);
+        if(!inputFileLocator.exists()){
+            logger.error(String.format("File %s does not exist", inputFileLocator.getAbsolutePath()));
             System.exit(-1);
         }
         File workingDirectory = new File(args[1]);
 
-        logger.info(String.format("File Name is %s", inputFile.getName()));
-        ResourceByteRangeProvider resourceByteRangeProvider = new FileByteRangeProvider(inputFile);
+        logger.info(String.format("File Name is %s", inputFileLocator.getName()));
+        ResourceByteRangeProvider resourceByteRangeProvider = inputFileLocator.getResourceByteRangeProvider();
         IMFTrackFileReader imfTrackFileReader = new IMFTrackFileReader(workingDirectory, resourceByteRangeProvider);
         StringBuilder sb = new StringBuilder();
         IMFErrorLogger imfErrorLogger = new IMFErrorLoggerImpl();
 
         try
         {
-            IMFTrackFileCPLBuilder imfTrackFileCPLBuilder = new IMFTrackFileCPLBuilder(workingDirectory, inputFile);
+            IMFTrackFileCPLBuilder imfTrackFileCPLBuilder = new IMFTrackFileCPLBuilder(workingDirectory, inputFileLocator);
             sb.append(imfTrackFileReader.getRandomIndexPack(imfErrorLogger));
             logger.info(String.format("%s", sb.toString()));
 

src/main/java/com/netflix/imflibrary/app/IMFTrackFileReader.java

@@ -26,19 +26,8 @@
 import com.netflix.imflibrary.st0377.PartitionPack;
 import com.netflix.imflibrary.st0377.RandomIndexPack;
 import com.netflix.imflibrary.st0377.StructuralMetadataID;
-import com.netflix.imflibrary.st0377.header.EssenceContainerData;
-import com.netflix.imflibrary.st0377.header.FileDescriptor;
-import com.netflix.imflibrary.st0377.header.GenericPackage;
-import com.netflix.imflibrary.st0377.header.InterchangeObject;
-import com.netflix.imflibrary.st0377.header.Preface;
-import com.netflix.imflibrary.st0377.header.SourcePackage;
-import com.netflix.imflibrary.utils.ByteArrayDataProvider;
-import com.netflix.imflibrary.utils.ByteProvider;
-import com.netflix.imflibrary.utils.ErrorLogger;
-import com.netflix.imflibrary.utils.FileByteRangeProvider;
-import com.netflix.imflibrary.utils.FileDataProvider;
-import com.netflix.imflibrary.utils.ResourceByteRangeProvider;
-import com.netflix.imflibrary.utils.Utilities;
+import com.netflix.imflibrary.st0377.header.*;
+import com.netflix.imflibrary.utils.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
@@ -62,7 +51,7 @@
  * A simple application to exercise the core logic of Photon for reading and validating IMF Track files.
  */
 @ThreadSafe
-final class IMFTrackFileReader
+final public class IMFTrackFileReader
 {
     private final File workingDirectory;
     private final ResourceByteRangeProvider resourceByteRangeProvider;
@@ -80,12 +69,18 @@ final class IMFTrackFileReader
      * @param workingDirectory the working directory
      * @param resourceByteRangeProvider the MXF file represented as a {@link com.netflix.imflibrary.utils.ResourceByteRangeProvider}
      */
-    IMFTrackFileReader(File workingDirectory, ResourceByteRangeProvider resourceByteRangeProvider)
+    public IMFTrackFileReader(File workingDirectory, ResourceByteRangeProvider resourceByteRangeProvider)
     {
         this.workingDirectory = workingDirectory;
         this.resourceByteRangeProvider = resourceByteRangeProvider;
     }
 
+    public IMFTrackFileReader(String inputFilePath, String workingDirectoryPath)
+    {
+        this.workingDirectory = new File(workingDirectoryPath);
+        this.resourceByteRangeProvider = FileLocator.fromLocation(inputFilePath).getResourceByteRangeProvider();
+    }
+
     private IMFConstraints.HeaderPartitionIMF getHeaderPartitionIMF(@Nonnull IMFErrorLogger imfErrorLogger) throws IOException
     {
         if (this.headerPartition == null)
@@ -592,7 +587,7 @@ BigInteger getEssenceEditRate(@Nonnull IMFErrorLogger imfErrorLogger) throws IOE
      * @param imfErrorLogger an error logger for recording any errors - cannot be null
      * @return editRate of the essence as a List of Long Integers
      */
-    List<Long> getEssenceEditRateAsList(@Nonnull IMFErrorLogger imfErrorLogger) throws IOException {
+    public List<Long> getEssenceEditRateAsList(@Nonnull IMFErrorLogger imfErrorLogger) throws IOException {
         if(!(this.getHeaderPartition(imfErrorLogger).getEssenceDescriptors().size() > 0)){
             throw new MXFException(String.format("No EssenceDescriptors were found in the MXF essence"));
         }
@@ -605,15 +600,15 @@ List<Long> getEssenceEditRateAsList(@Nonnull IMFErrorLogger imfErrorLogger) thro
      * @param imfErrorLogger an error logger for recording any errors - cannot be null
      * @return essenceDuration
      */
-    BigInteger getEssenceDuration(@Nonnull IMFErrorLogger imfErrorLogger) throws IOException {
+    public BigInteger getEssenceDuration(@Nonnull IMFErrorLogger imfErrorLogger) throws IOException {
         return this.getHeaderPartition(imfErrorLogger).getEssenceDuration();
     }
 
     /**
      * A method to return the TrackFileId which is a UUID identifying the track file
      * @return UUID identifying the Track File
      */
-    UUID getTrackFileId(){
+    public UUID getTrackFileId(){
 
         Preface preface = this.headerPartition.getHeaderPartitionOP1A().getHeaderPartition().getPreface();
         GenericPackage genericPackage = preface.getContentStorage().getEssenceContainerDataList().get(0).getLinkedPackage();
@@ -680,20 +675,20 @@ public static void main(String[] args) throws IOException
             throw new IllegalArgumentException("Invalid parameters");
         }
 
-        File inputFile = new File(args[0]);
-        if(!inputFile.exists()){
-            logger.error(String.format("File %s does not exist", inputFile.getAbsolutePath()));
+        FileLocator inputFileLocator = FileLocator.fromLocation(args[0]);
+        if(!inputFileLocator.exists()){
+            logger.error(String.format("File %s does not exist", inputFileLocator.getAbsolutePath()));
             System.exit(-1);
         }
         File workingDirectory = new File(args[1]);
 
-        ResourceByteRangeProvider resourceByteRangeProvider = new FileByteRangeProvider(inputFile);
+        ResourceByteRangeProvider resourceByteRangeProvider = inputFileLocator.getResourceByteRangeProvider();
         IMFTrackFileReader imfTrackFileReader = null;
         IMFTrackFileCPLBuilder imfTrackFileCPLBuilder = null;
         IMFErrorLogger imfErrorLogger = new IMFErrorLoggerImpl();
         try {
-            imfTrackFileReader = new IMFTrackFileReader(workingDirectory, resourceByteRangeProvider);
-            imfTrackFileCPLBuilder = new IMFTrackFileCPLBuilder(workingDirectory, inputFile);
+            imfTrackFileReader = new IMFTrackFileReader(args[0], args[1]);
+            imfTrackFileCPLBuilder = new IMFTrackFileCPLBuilder(workingDirectory, inputFileLocator);
         }
         catch (IMFException | MXFException e){
             if(e instanceof IMFException){
@@ -714,6 +709,14 @@ else if(e instanceof MXFException){
                 && imfTrackFileCPLBuilder != null
                 && supportedEssenceComponentTypes.contains(imfTrackFileReader.getEssenceType(imfErrorLogger))) {
             try {
+                HeaderPartition headerPartition = imfTrackFileReader.headerPartition.getHeaderPartitionOP1A().getHeaderPartition();
+                List<InterchangeObject.InterchangeObjectBO> subDescriptors = headerPartition.getSubDescriptors();
+                for (InterchangeObject.InterchangeObjectBO subDescriptor : subDescriptors) {
+                    if (subDescriptor instanceof PHDRMetaDataTrackSubDescriptor.PHDRMetaDataTrackSubDescriptorBO) {
+                        logger.info("Found a PHDRMetaDataTrackSubDescriptor with instanceID: " + UUIDHelper.fromUUID(UUID.nameUUIDFromBytes(subDescriptor.getInstanceUID().getUID())));
+                    }
+                }
+
                 for (InterchangeObject.InterchangeObjectBO essenceDescriptor : imfTrackFileReader.getEssenceDescriptors(imfErrorLogger)) {
                 /* create dom */
                     DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();