[Proposal] scte214 - use supplemental codec instead of base codec if it's supported

src/manifest/representation.ts

@@ -111,6 +111,20 @@ class Representation {
    */
   public codec : string | undefined;
 
+  /**
+   * Supplemental codecs are defined as backwards-compatible codecs enhancing
+   * the experience of a base layer codec
+   * Optional attribute, a representation may not have supplemental codecs
+   */
+  public supplementalCodec?: string | undefined;
+
+  /**
+   * `true` if the Supplemental codec is in a supported codec
+   * `false` if there is no supplemental codec or
+   * if the supplemental codec is not supported
+   */
+  public isSupplementalCodecSupported: boolean;
+
   /**
    * A string describing the mime-type for this Representation.
    * Examples: audio/mp4, video/webm, application/mp4, text/plain
@@ -191,8 +205,26 @@ class Representation {
 
     this.cdnMetadata = args.cdnMetadata;
 
+    if (args.supplementalCodecs !== undefined) {
+      this.supplementalCodec = args.supplementalCodecs;
+    }
+
+    if (this.supplementalCodec !== undefined) {
+      const supplementalCodecMimeTypeStr =
+       `${this.mimeType ?? ""};codecs="${this.supplementalCodec ?? ""}"`;
+      const isSupplementalCodecSupported = isCodecSupported(supplementalCodecMimeTypeStr);
+      this.isSupplementalCodecSupported = isSupplementalCodecSupported;
+    } else {
+      this.isSupplementalCodecSupported = false;
+    }
+
     this.index = args.index;
-    if (opts.type === "audio" || opts.type === "video") {
+
+    if (this.isSupplementalCodecSupported) {
+    // The supplemental codec being supported indicate that the base codec will also
+    // be supported as the supplemental codec is backwards compatible with base codec
+      this.isSupported = true;
+    } else if (opts.type === "audio" || opts.type === "video") {
       const mimeTypeStr = this.getMimeTypeString();
       const isSupported = isCodecSupported(mimeTypeStr);
       if (!isSupported) {
@@ -210,7 +242,11 @@ class Representation {
    * @returns {string}
    */
   public getMimeTypeString() : string {
-    return `${this.mimeType ?? ""};codecs="${this.codec ?? ""}"`;
+    if (this.isSupplementalCodecSupported) {
+      return `${this.mimeType ?? ""};codecs="${this.supplementalCodec ?? ""}"`;
+    } else {
+      return `${this.mimeType ?? ""};codecs="${this.codec ?? ""}"`;
+    }
   }
 
   /**
@@ -387,17 +423,38 @@ class Representation {
    * @returns {Object}
    */
   public toAudioRepresentation(): IAudioRepresentation {
-    const { id, isSpatialAudio, bitrate, codec } = this;
-    return { id, isSpatialAudio, bitrate, codec };
+    const { id,
+            isSpatialAudio,
+            bitrate,
+            codec,
+            supplementalCodec,
+            isSupplementalCodecSupported,
+    } = this;
+    // Depending if the device can play the supplemental codec or not
+    // We return as codec the codec that WILL be played if this representation is chosen
+    const codecInUse = isSupplementalCodecSupported ? supplementalCodec : codec;
+    return { id, isSpatialAudio, bitrate, codec: codecInUse };
   }
 
   /**
    * Format Representation as an `IVideoRepresentation`.
    * @returns {Object}
    */
   public toVideoRepresentation(): IVideoRepresentation {
-    const { id, bitrate, frameRate, width, height, codec, hdrInfo } = this;
-    return { id, bitrate, frameRate, width, height, codec, hdrInfo };
+    const { id,
+            bitrate,
+            frameRate,
+            width,
+            height,
+            codec,
+            hdrInfo,
+            supplementalCodec,
+            isSupplementalCodecSupported,
+    } = this;
+    // Depending if the device can play the supplemental codec or not
+    // We return as codec the codec that WILL be played if this representation is chosen
+    const codecInUse = isSupplementalCodecSupported ? supplementalCodec : codec;
+    return { id, bitrate, frameRate, width, height, codec: codecInUse, hdrInfo };
   }
 }
 

src/parsers/manifest/dash/common/indexes/__tests__/utils.test.ts

@@ -0,0 +1,37 @@
+import { convertSupplementalCodecsToRFC6381 } from "../utils";
+
+describe("parseSupplementalCodec", () => {
+  it("should return the codec unchanged if there is only one codec", () => {
+    expect(convertSupplementalCodecsToRFC6381("avc1.4d400d"))
+      .toEqual("avc1.4d400d");
+  });
+  it("should trim starting and ending whitespace", () => {
+    expect(
+      convertSupplementalCodecsToRFC6381("  avc1.4d400d  "))
+      .toEqual("avc1.4d400d");
+  });
+  it("should return comma-separated list if input is whitespace-separated", () => {
+    expect(
+      convertSupplementalCodecsToRFC6381("avc1.4d400d avc1.4d4015"))
+      .toEqual("avc1.4d400d, avc1.4d4015");
+  });
+  it("should return comma-separated value if input is already comma-separated", () => {
+    expect(
+      convertSupplementalCodecsToRFC6381("avc1.4d400d, avc1.4d4015"))
+      .toEqual("avc1.4d400d, avc1.4d4015");
+  });
+
+  it("should return comma-separated value if input as missplaced whitespace", () => {
+    expect(
+      convertSupplementalCodecsToRFC6381("avc1.4d400d  ,  avc1.4d4015 "))
+      .toEqual("avc1.4d400d, avc1.4d4015");
+  });
+
+  it(`should return comma-separated value if input is mix of comma and 
+    whitespace separated list`
+  , () => {
+    expect(
+      convertSupplementalCodecsToRFC6381("avc1.4d400d avc1.4d4015, avc1.4d401f"))
+      .toEqual("avc1.4d400d, avc1.4d4015, avc1.4d401f");
+  });
+});

src/parsers/manifest/dash/common/indexes/utils.ts

@@ -15,6 +15,7 @@
  */
 
 import config from "../../../../../config";
+import isNonEmptyString from "../../../../../utils/is_non_empty_string";
 
 
 /**
@@ -30,3 +31,33 @@ export function getSegmentTimeRoundingError(timescale: number): number {
   return config.getCurrent().DEFAULT_MAXIMUM_TIME_ROUNDING_ERROR * timescale;
 }
 
+const supplementalCodecSeparator = /[, ]+/g;
+/**
+ * Converts SCTE 214 supplemental codec string into RFC4281 codec string
+ *
+ * The returned value is a codec string respecting RFC6381
+ *
+ * SCTE 214 defines supplemental codecs as a whitespace-separated multiple list of
+ * codec strings
+ *
+ * RFC6381 defines codecs as a comma-separated list of codec strings.
+ *
+ * This two syntax differs and this parser is used to convert SCTE214
+ * to be compliant with what MSE APIs expect
+ *
+ * @param {string} val - The codec string to parse
+ * @returns { Array.<string | undefined | null>}
+ */
+export function convertSupplementalCodecsToRFC6381(
+  val: string
+) : string {
+
+  if (isNonEmptyString(val)) {
+    return val
+      .trim()
+      .replace(supplementalCodecSeparator, ", ");
+  }
+  return "";
+}
+
+

src/parsers/manifest/dash/common/parse_representations.ts

@@ -18,6 +18,7 @@ import log from "../../../../log";
 import { Adaptation } from "../../../../manifest";
 import { IHDRInformation } from "../../../../public_types";
 import arrayFind from "../../../../utils/array_find";
+import isNullOrUndefined from "../../../../utils/is_null_or_undefined";
 import objectAssign from "../../../../utils/object_assign";
 import {
   IContentProtections,
@@ -30,6 +31,7 @@ import {
   IContentProtectionIntermediateRepresentation,
 } from "../node_parser_types";
 import { getWEBMHDRInformation } from "./get_hdr_information";
+import { convertSupplementalCodecsToRFC6381 } from "./indexes/utils";
 import parseRepresentationIndex, {
   IRepresentationIndexContext,
 } from "./parse_representation_index";
@@ -120,19 +122,19 @@ export default function parseRepresentations(
   const parsedRepresentations : IParsedRepresentation[] = [];
   for (const representation of representationsIR) {
     // Compute Representation ID
-    let representationID = representation.attributes.id != null ?
+    let representationID = !isNullOrUndefined(representation.attributes.id) ?
       representation.attributes.id :
       (String(representation.attributes.bitrate) +
-         (representation.attributes.height != null ?
+         (!isNullOrUndefined(representation.attributes.height) ?
             (`-${representation.attributes.height}`) :
             "") +
-         (representation.attributes.width != null ?
+         (!isNullOrUndefined(representation.attributes.width) ?
             (`-${representation.attributes.width}`) :
             "") +
-         (representation.attributes.mimeType != null ?
+         (!isNullOrUndefined(representation.attributes.mimeType) ?
             (`-${representation.attributes.mimeType}`) :
             "") +
-         (representation.attributes.codecs != null ?
+         (!isNullOrUndefined(representation.attributes.codecs) ?
             (`-${representation.attributes.codecs}`) :
             ""));
 
@@ -167,7 +169,7 @@ export default function parseRepresentations(
 
     // Find bitrate
     let representationBitrate : number;
-    if (representation.attributes.bitrate == null) {
+    if (isNullOrUndefined(representation.attributes.bitrate)) {
       log.warn("DASH: No usable bitrate found in the Representation.");
       representationBitrate = 0;
     } else {
@@ -204,40 +206,52 @@ export default function parseRepresentations(
 
     // Add optional attributes
     let codecs : string|undefined;
-    if (representation.attributes.codecs != null) {
+    if (!isNullOrUndefined(representation.attributes.codecs)) {
       codecs = representation.attributes.codecs;
-    } else if (adaptation.attributes.codecs != null) {
+    } else if (!isNullOrUndefined(adaptation.attributes.codecs)) {
       codecs = adaptation.attributes.codecs;
     }
-    if (codecs != null) {
+    if (!isNullOrUndefined(codecs)) {
       codecs = codecs === "mp4a.40.02" ? "mp4a.40.2" : codecs;
       parsedRepresentation.codecs = codecs;
     }
-    if (representation.attributes.frameRate != null) {
+
+    let supplementalCodecs: string | undefined;
+    if (!isNullOrUndefined(representation.attributes.supplementalCodecs)) {
+      supplementalCodecs = representation.attributes.supplementalCodecs;
+    } else if (!isNullOrUndefined(adaptation.attributes.supplementalCodecs)) {
+      supplementalCodecs = adaptation.attributes.supplementalCodecs;
+    }
+    if (!isNullOrUndefined(supplementalCodecs)) {
+      parsedRepresentation.supplementalCodecs =
+        convertSupplementalCodecsToRFC6381(supplementalCodecs);
+    }
+
+    if (!isNullOrUndefined(representation.attributes.frameRate)) {
       parsedRepresentation.frameRate =
         representation.attributes.frameRate;
-    } else if (adaptation.attributes.frameRate != null) {
+    } else if (!isNullOrUndefined(adaptation.attributes.frameRate)) {
       parsedRepresentation.frameRate =
         adaptation.attributes.frameRate;
     }
-    if (representation.attributes.height != null) {
+    if (!isNullOrUndefined(representation.attributes.height)) {
       parsedRepresentation.height =
         representation.attributes.height;
-    } else if (adaptation.attributes.height != null) {
+    } else if (!isNullOrUndefined(adaptation.attributes.height)) {
       parsedRepresentation.height =
         adaptation.attributes.height;
     }
-    if (representation.attributes.mimeType != null) {
+    if (!isNullOrUndefined(representation.attributes.mimeType)) {
       parsedRepresentation.mimeType =
         representation.attributes.mimeType;
-    } else if (adaptation.attributes.mimeType != null) {
+    } else if (!isNullOrUndefined(adaptation.attributes.mimeType)) {
       parsedRepresentation.mimeType =
         adaptation.attributes.mimeType;
     }
-    if (representation.attributes.width != null) {
+    if (!isNullOrUndefined(representation.attributes.width)) {
       parsedRepresentation.width =
         representation.attributes.width;
-    } else if (adaptation.attributes.width != null) {
+    } else if (!isNullOrUndefined(adaptation.attributes.width)) {
       parsedRepresentation.width =
         adaptation.attributes.width;
     }

src/parsers/manifest/dash/js-parser/node_parsers/AdaptationSet.ts

@@ -296,6 +296,10 @@ function parseAdaptationSetAttributes(
         parsedAdaptation.codecs = attribute.value;
         break;
 
+      case "scte214:supplementalCodecs":
+        parsedAdaptation.supplementalCodecs = attribute.value;
+        break;
+
       case "codingDependency":
         parseValue(attribute.value, { asKey: "codingDependency",
                                       parser: parseBoolean,

src/parsers/manifest/dash/js-parser/node_parsers/Representation.ts

@@ -184,6 +184,10 @@ function parseRepresentationAttributes(
                                  dashName: "qualityRanking" });
         break;
 
+      case "scte214:supplementalCodecs":
+        attributes.supplementalCodecs = attr.value;
+        break;
+
       case "segmentProfiles":
         attributes.segmentProfiles = attr.value;
         break;

src/parsers/manifest/dash/js-parser/node_parsers/utils.ts

@@ -28,6 +28,7 @@ const iso8601Duration =
   /^P(([\d.]*)Y)?(([\d.]*)M)?(([\d.]*)D)?T?(([\d.]*)H)?(([\d.]*)M)?(([\d.]*)S)?/;
 const rangeRe = /([0-9]+)-([0-9]+)/;
 
+
 /**
  * Parse MPD boolean attributes.
  *

src/parsers/manifest/dash/node_parser_types.ts

@@ -232,6 +232,7 @@ export interface IAdaptationSetAttributes {
   segmentAlignment? : number|boolean;
   segmentProfiles? : string;
   subsegmentAlignment? : number|boolean;
+  supplementalCodecs?: string;
   width? : number;
   availabilityTimeComplete?: boolean;
   availabilityTimeOffset?: number;
@@ -271,6 +272,7 @@ export interface IRepresentationAttributes {
   profiles? : string;
   qualityRanking? : number;
   segmentProfiles? : string;
+  supplementalCodecs?: string;
   width? : number;
   availabilityTimeComplete?: boolean;
   availabilityTimeOffset?: number;

src/parsers/manifest/dash/wasm-parser/rs/events.rs

@@ -280,6 +280,8 @@ pub enum AttributeName {
 
     // SegmentTemplate
     EndNumber = 76, // f64
+
+    SupplementalCodecs = 77 // string
 }
 
 impl TagName {

src/parsers/manifest/dash/wasm-parser/rs/processor/attributes.rs

@@ -87,6 +87,7 @@ pub fn report_adaptation_set_attrs(e: &quick_xml::events::BytesStart) {
                 b"bitstreamSwitching" => BitstreamSwitching.try_report_as_bool(&attr),
                 b"audioSamplingRate" => AudioSamplingRate.try_report_as_string(&attr),
                 b"codecs" => Codecs.try_report_as_string(&attr),
+                b"scte214:supplementalCodecs" => SupplementalCodecs.try_report_as_string(&attr),
                 b"profiles" => Profiles.try_report_as_string(&attr),
                 b"segmentProfiles" => SegmentProfiles.try_report_as_string(&attr),
                 b"mimeType" => MimeType.try_report_as_string(&attr),
@@ -116,6 +117,7 @@ pub fn report_representation_attrs(tag_bs: &quick_xml::events::BytesStart) {
                 b"audioSamplingRate" => AudioSamplingRate.try_report_as_string(&attr),
                 b"bandwidth" => Bitrate.try_report_as_u64(&attr),
                 b"codecs" => Codecs.try_report_as_string(&attr),
+                b"scte214:supplementalCodecs" => SupplementalCodecs.try_report_as_string(&attr),
                 b"codingDependency" => CodingDependency.try_report_as_bool(&attr),
                 b"frameRate" => FrameRate.try_report_as_string(&attr),
                 b"height" => Height.try_report_as_u64(&attr),

src/parsers/manifest/dash/wasm-parser/ts/generators/AdaptationSet.ts

@@ -289,6 +289,10 @@ export function generateAdaptationSetAttrParser(
         adaptationAttrs.codecs =
           parseString(textDecoder, linearMemory.buffer, ptr, len);
         break;
+      case AttributeName.SupplementalCodecs:
+        adaptationAttrs.supplementalCodecs =
+          parseString(textDecoder, linearMemory.buffer, ptr, len);
+        break;
       case AttributeName.Profiles:
         adaptationAttrs.profiles =
           parseString(textDecoder, linearMemory.buffer, ptr, len);

src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts

@@ -165,6 +165,10 @@ export function generateRepresentationAttrParser(
         representationAttrs.codecs =
           parseString(textDecoder, linearMemory.buffer, ptr, len);
         break;
+      case AttributeName.SupplementalCodecs:
+        representationAttrs.supplementalCodecs =
+        parseString(textDecoder, linearMemory.buffer, ptr, len);
+        break;
       case AttributeName.CodingDependency:
         representationAttrs.codingDependency =
           new DataView(linearMemory.buffer).getUint8(0) === 0;

src/parsers/manifest/dash/wasm-parser/ts/types.ts

@@ -293,4 +293,6 @@ export const enum AttributeName {
 
   // SegmentTemplate
   EndNumber = 76, // f64
+
+  SupplementalCodecs = 77, // String
 }