Cherry pick PR #3500: [android] Refactor audio MediaCodecBridge creat…

…ion (#3524) Refer to the original PR: #3500 Moved MediaCodecBridge.createAudioMediaCodecBridge() to MediaCodecBridge.createAudioDecoder(), and refactored opus and aac configuration setting. b/345542000 Change-Id: Idac359d9b8472b62d41d0a16192db772ee5d8011 Co-authored-by: xiaomings <[email protected]>
youtube · Jun 11, 2024 · 9ba2896 · 9ba2896
1 parent 50fa91f
commit 9ba2896
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 134 deletions.
diff --git a/starboard/android/apk/apk_sources.gni b/starboard/android/apk/apk_sources.gni
@@ -44,6 +44,7 @@ apk_sources = [
   "//starboard/android/apk/app/src/main/java/dev/cobalt/media/CobaltMediaSession.java",
   "//starboard/android/apk/app/src/main/java/dev/cobalt/media/Log.java",
   "//starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java",
+  "//starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridgeBuilder.java",
   "//starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecCapabilitiesLogger.java",
   "//starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecUtil.java",
   "//starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaDrmBridge.java",

diff --git a/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java b/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java
@@ -32,7 +32,6 @@
 import android.os.Build;
 import android.os.Bundle;
 import android.view.Surface;
-import androidx.annotation.Nullable;
 import dev.cobalt.util.Log;
 import dev.cobalt.util.UsedByNative;
 import java.nio.ByteBuffer;
@@ -446,7 +445,7 @@ private String errorMessage() {
     }
   }
 
-  private MediaCodecBridge(
+  public MediaCodecBridge(
       long nativeMediaCodecBridge,
       MediaCodec mediaCodec,
       String mime,
@@ -552,61 +551,6 @@ public static boolean isFrameRenderedCallbackEnabled() {
     return Build.VERSION.SDK_INT >= 34;
   }
 
-  @SuppressWarnings("unused")
-  @UsedByNative
-  public static MediaCodecBridge createAudioMediaCodecBridge(
-      long nativeMediaCodecBridge,
-      String mime,
-      String decoderName,
-      int sampleRate,
-      int channelCount,
-      MediaCrypto crypto,
-      @Nullable byte[] configurationData) {
-    if (decoderName.equals("")) {
-      Log.e(TAG, "Invalid decoder name.");
-      return null;
-    }
-    MediaCodec mediaCodec = null;
-    try {
-      Log.i(TAG, "Creating \"%s\" decoder.", decoderName);
-      mediaCodec = MediaCodec.createByCodecName(decoderName);
-    } catch (Exception e) {
-      Log.e(TAG, "Failed to create MediaCodec: %s, DecoderName: %s", mime, decoderName, e);
-      return null;
-    }
-    if (mediaCodec == null) {
-      return null;
-    }
-    MediaCodecBridge bridge =
-        new MediaCodecBridge(
-            nativeMediaCodecBridge, mediaCodec, mime, BitrateAdjustmentTypes.NO_ADJUSTMENT, -1);
-
-    MediaFormat mediaFormat = createAudioFormat(mime, sampleRate, channelCount);
-
-    if (mime.contains("opus")) {
-      if (!setOpusConfigurationData(mediaFormat, sampleRate, configurationData)) {
-        bridge.release();
-        return null;
-      }
-    } else {
-      // TODO: Determine if we should explicitly check the mime for AAC audio before calling
-      // setFrameHasADTSHeader(), as more codecs may be supported here in the future.
-      setFrameHasADTSHeader(mediaFormat);
-    }
-    if (!bridge.configureAudio(mediaFormat, crypto, 0)) {
-      Log.e(TAG, "Failed to configure audio codec.");
-      bridge.release();
-      return null;
-    }
-    if (!bridge.start()) {
-      Log.e(TAG, "Failed to start audio codec.");
-      bridge.release();
-      return null;
-    }
-
-    return bridge;
-  }
-
   @SuppressWarnings("unused")
   @UsedByNative
   public static void createVideoMediaCodecBridge(
@@ -834,7 +778,7 @@ public void release() {
 
   @SuppressWarnings("unused")
   @UsedByNative
-  private boolean start() {
+  public boolean start() {
     try {
       mMediaCodec.start();
     } catch (IllegalStateException | IllegalArgumentException e) {
@@ -1130,10 +1074,6 @@ private boolean configureVideo(
     return false;
   }
 
-  public static MediaFormat createAudioFormat(String mime, int sampleRate, int channelCount) {
-    return MediaFormat.createAudioFormat(mime, sampleRate, channelCount);
-  }
-
   private static MediaFormat createVideoDecoderFormat(
       String mime, int widthHint, int heightHint, VideoCapabilities videoCapabilities) {
     return MediaFormat.createVideoFormat(
@@ -1216,55 +1156,9 @@ private void maybeSetMaxVideoInputSize(MediaFormat format) {
     }
   }
 
-  @SuppressWarnings("unused")
-  private static boolean setOpusConfigurationData(
-      MediaFormat format, int sampleRate, @Nullable byte[] configurationData) {
-    final int MIN_OPUS_INITIALIZATION_DATA_BUFFER_SIZE = 19;
-    final long NANOSECONDS_IN_ONE_SECOND = 1000000000L;
-    // 3840 is the default seek pre-roll samples used by ExoPlayer:
-    // https://github.com/google/ExoPlayer/blob/0ba317b1337eaa789f05dd6c5241246478a3d1e5/library/common/src/main/java/com/google/android/exoplayer2/audio/OpusUtil.java#L30.
-    final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840;
-    if (configurationData == null
-        || configurationData.length < MIN_OPUS_INITIALIZATION_DATA_BUFFER_SIZE) {
-      Log.e(
-          TAG,
-          "Failed to configure Opus audio codec. "
-              + (configurationData == null
-                  ? "|configurationData| is null."
-                  : String.format(
-                      Locale.US,
-                      "Configuration data size (%d) is less than the required size (%d).",
-                      configurationData.length,
-                      MIN_OPUS_INITIALIZATION_DATA_BUFFER_SIZE)));
-      return false;
-    }
-    // Both the number of samples to skip from the beginning of the stream and the amount of time
-    // to pre-roll when seeking must be specified when configuring the Opus decoder. Logic adapted
-    // from ExoPlayer:
-    // https://github.com/google/ExoPlayer/blob/0ba317b1337eaa789f05dd6c5241246478a3d1e5/library/common/src/main/java/com/google/android/exoplayer2/audio/OpusUtil.java#L52.
-    int preSkipSamples = ((configurationData[11] & 0xFF) << 8) | (configurationData[10] & 0xFF);
-    long preSkipNanos = (preSkipSamples * NANOSECONDS_IN_ONE_SECOND) / sampleRate;
-    long seekPreRollNanos =
-        (DEFAULT_SEEK_PRE_ROLL_SAMPLES * NANOSECONDS_IN_ONE_SECOND) / sampleRate;
-    MediaFormatBuilder.setCodecSpecificData(
-        format,
-        new byte[][] {
-          configurationData,
-          ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(preSkipNanos).array(),
-          ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(seekPreRollNanos).array(),
-        });
-    return true;
-  }
-
-  @SuppressWarnings("unused")
-  @UsedByNative
-  private static void setFrameHasADTSHeader(MediaFormat format) {
-    format.setInteger(MediaFormat.KEY_IS_ADTS, 1);
-  }
-
   @SuppressWarnings("unused")
   @UsedByNative
-  private boolean configureAudio(MediaFormat format, MediaCrypto crypto, int flags) {
+  public boolean configureAudio(MediaFormat format, MediaCrypto crypto, int flags) {
     try {
       mMediaCodec.configure(format, null, crypto, flags);
       return true;

diff --git a/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridgeBuilder.java b/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridgeBuilder.java
@@ -18,11 +18,19 @@
 
 package dev.cobalt.media;
 
-/*
+import static dev.cobalt.media.Log.TAG;
+
+import android.media.MediaCodec;
+import android.media.MediaCrypto;
+import android.media.MediaFormat;
+import androidx.annotation.Nullable;
+import dev.cobalt.util.Log;
+import dev.cobalt.util.UsedByNative;
+
 class MediaCodecBridgeBuilder {
   @SuppressWarnings("unused")
   @UsedByNative
-  public static MediaCodecBridge createAudioMediaCodecBridge(
+  public static MediaCodecBridge createAudioDecoder(
       long nativeMediaCodecBridge,
       String mime,
       String decoderName,
@@ -47,20 +55,29 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
     }
     MediaCodecBridge bridge =
         new MediaCodecBridge(
-            nativeMediaCodecBridge, mediaCodec, mime, BitrateAdjustmentTypes.NO_ADJUSTMENT, -1);
-
-    MediaFormat mediaFormat = createAudioFormat(mime, sampleRate, channelCount);
+            nativeMediaCodecBridge,
+            mediaCodec,
+            mime,
+            MediaCodecBridge.BitrateAdjustmentTypes.NO_ADJUSTMENT,
+            -1);
 
+    byte[][] csds = {};
+    boolean frameHasAdtsHeader = false;
     if (mime.contains("opus")) {
-      if (!setOpusConfigurationData(mediaFormat, sampleRate, configurationData)) {
+      csds = MediaFormatBuilder.starboardParseOpusConfigurationData(sampleRate, configurationData);
+      if (csds == null) {
         bridge.release();
         return null;
       }
     } else {
-      // TODO: Determine if we should explicitly check the mime for AAC audio before calling
-      // setFrameHasADTSHeader(), as more codecs may be supported here in the future.
-      setFrameHasADTSHeader(mediaFormat);
+      // TODO: Determine if we should explicitly check the mime for AAC audio before setting
+      // frameHasAdtsHeader to true, as more codecs may be supported here in the future.
+      frameHasAdtsHeader = true;
     }
+    MediaFormat mediaFormat =
+        MediaFormatBuilder.createAudioFormat(
+            mime, sampleRate, channelCount, csds, frameHasAdtsHeader);
+
     if (!bridge.configureAudio(mediaFormat, crypto, 0)) {
       Log.e(TAG, "Failed to configure audio codec.");
       bridge.release();
@@ -75,4 +92,3 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
     return bridge;
   }
 }
-*/
diff --git a/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaFormatBuilder.java b/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaFormatBuilder.java
@@ -18,8 +18,14 @@
 
 package dev.cobalt.media;
 
+import static dev.cobalt.media.Log.TAG;
+
 import android.media.MediaFormat;
+import androidx.annotation.Nullable;
+import dev.cobalt.util.Log;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Locale;
 
 class MediaFormatBuilder {
   public static void setCodecSpecificData(MediaFormat format, byte[][] csds) {
@@ -33,10 +39,18 @@ public static void setCodecSpecificData(MediaFormat format, byte[][] csds) {
     }
   }
 
-  /*
-  @SuppressWarnings("unused")
-  private static boolean setOpusConfigurationData(
-      MediaFormat format, int sampleRate, @Nullable byte[] configurationData) {
+  public static MediaFormat createAudioFormat(
+      String mime, int sampleRate, int channelCount, byte[][] csds, boolean frameHasAdtsHeader) {
+    MediaFormat format = MediaFormat.createAudioFormat(mime, sampleRate, channelCount);
+    setCodecSpecificData(format, csds);
+    if (frameHasAdtsHeader) {
+      format.setInteger(MediaFormat.KEY_IS_ADTS, 1);
+    }
+    return format;
+  }
+
+  public static byte[][] starboardParseOpusConfigurationData(
+      int sampleRate, @Nullable byte[] configurationData) {
     final int MIN_OPUS_INITIALIZATION_DATA_BUFFER_SIZE = 19;
     final long NANOSECONDS_IN_ONE_SECOND = 1000000000L;
     // 3840 is the default seek pre-roll samples used by ExoPlayer:
@@ -54,7 +68,7 @@ private static boolean setOpusConfigurationData(
                       "Configuration data size (%d) is less than the required size (%d).",
                       configurationData.length,
                       MIN_OPUS_INITIALIZATION_DATA_BUFFER_SIZE)));
-      return false;
+      return null;
     }
     // Both the number of samples to skip from the beginning of the stream and the amount of time
     // to pre-roll when seeking must be specified when configuring the Opus decoder. Logic adapted
@@ -64,14 +78,10 @@ private static boolean setOpusConfigurationData(
     long preSkipNanos = (preSkipSamples * NANOSECONDS_IN_ONE_SECOND) / sampleRate;
     long seekPreRollNanos =
         (DEFAULT_SEEK_PRE_ROLL_SAMPLES * NANOSECONDS_IN_ONE_SECOND) / sampleRate;
-    MediaFormatBuilder.setCodecSpecificData(
-        format,
-        new byte[][] {
-          configurationData,
-          ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(preSkipNanos).array(),
-          ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(seekPreRollNanos).array(),
-        });
-    return true;
+    return new byte[][] {
+      configurationData,
+      ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(preSkipNanos).array(),
+      ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(seekPreRollNanos).array(),
+    };
   }
-  */
 }
diff --git a/starboard/android/shared/media_codec_bridge.cc b/starboard/android/shared/media_codec_bridge.cc
@@ -192,7 +192,7 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
   scoped_ptr<MediaCodecBridge> native_media_codec_bridge(
       new MediaCodecBridge(handler));
   jobject j_media_codec_bridge = env->CallStaticObjectMethodOrAbort(
-      "dev/cobalt/media/MediaCodecBridge", "createAudioMediaCodecBridge",
+      "dev/cobalt/media/MediaCodecBridgeBuilder", "createAudioDecoder",
       "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;"
       "[B)Ldev/cobalt/media/MediaCodecBridge;",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),