feat(*): add support for HDR

ThibaultBee · Dec 9, 2023 · 6ede1af · 6ede1af
1 parent 68f5d67
commit 6ede1af
Show file tree

Hide file tree

Showing 23 changed files with 569 additions and 87 deletions.
diff --git a/core/src/main/java/io/github/thibaultbee/streampack/data/Config.kt b/core/src/main/java/io/github/thibaultbee/streampack/data/Config.kt
@@ -41,7 +41,7 @@ open class Config(
 
     /**
      * The encoder profile.
-     * Only applicable to AAC, AVC and HEVC.
+     * Only applicable to AAC, AVC, HEVC, VP9, AV1.
      */
     val profile: Int = 0
 ) {
@@ -75,6 +75,7 @@ open class Config(
      * Check if this configuration is supported by the specified encoder.
      * If format is not supported, it won't be possible to start a stream.
      *
+     * @param name the encoder name
      * @return true if format is supported, otherwise false
      */
     fun isFormatSupportedForEncoder(name: String): Boolean {
@@ -91,7 +92,7 @@ open class Config(
      *
      * @return the default encoder name
      */
-    val defaultEncodeName: String? by lazy {
+    val defaultEncoderName: String? by lazy {
         try {
             MediaCodecHelper.findEncoder(getFormat(true))
         } catch (_: Exception) {

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt b/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt
@@ -16,6 +16,7 @@
 package io.github.thibaultbee.streampack.data
 
 import android.content.Context
+import android.media.MediaCodecInfo
 import android.media.MediaCodecInfo.CodecProfileLevel
 import android.media.MediaCodecInfo.CodecProfileLevel.AV1ProfileMain8
 import android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline
@@ -26,12 +27,12 @@ import android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileHigh
 import android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileMain
 import android.media.MediaCodecInfo.CodecProfileLevel.HEVCProfileMain
 import android.media.MediaCodecInfo.CodecProfileLevel.VP9Profile0
-import android.media.MediaCodecInfo.CodecProfileLevel.VP9Profile1
 import android.media.MediaFormat
 import android.media.MediaFormat.KEY_PRIORITY
 import android.os.Build
 import android.util.Size
 import io.github.thibaultbee.streampack.internal.encoders.MediaCodecHelper
+import io.github.thibaultbee.streampack.internal.utils.av.video.DynamicRangeProfile
 import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait
 import io.github.thibaultbee.streampack.internal.utils.extensions.isVideo
 import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize
@@ -70,6 +71,9 @@ class VideoConfig(
     val fps: Int = 30,
     /**
      * Video encoder profile. Encoders may not support requested profile. In this case, StreamPack fallbacks to default profile.
+     * If not set, profile is always a 8 bit profile. StreamPack try to apply the highest profile available.
+     * If the decoder does not support the profile, you should explicitly set the profile to a lower
+     * value such as [AVCProfileBaseline] for AVC, [HEVCProfileMain] for HEVC, [VP9Profile0] for VP9.
      * ** See ** [MediaCodecInfo.CodecProfileLevel](https://developer.android.com/reference/android/media/MediaCodecInfo.CodecProfileLevel)
      */
     profile: Int = getBestProfile(mimeType),
@@ -94,7 +98,7 @@ class VideoConfig(
     constructor(
         /**
          * Video encoder mime type.
-         * Only [MediaFormat.MIMETYPE_VIDEO_AVC], [MediaFormat.MIMETYPE_VIDEO_HEVC] and [MediaFormat.MIMETYPE_VIDEO_VP9] are supported yet.
+         * Only [MediaFormat.MIMETYPE_VIDEO_AVC], [MediaFormat.MIMETYPE_VIDEO_HEVC], [MediaFormat.MIMETYPE_VIDEO_VP9] and [MediaFormat.MIMETYPE_VIDEO_AV1] are supported yet.
          *
          * **See Also:** [MediaFormat MIMETYPE_VIDEO_*](https://developer.android.com/reference/android/media/MediaFormat)
          */
@@ -133,6 +137,13 @@ class VideoConfig(
         gopDuration
     )
 
+    /**
+     * The dynamic range profile.
+     * It is deduced from the [profile].
+     * **See Also:** [DynamicRangeProfiles](https://developer.android.com/reference/android/hardware/camera2/params/DynamicRangeProfiles)
+     */
+    val dynamicRangeProfile = DynamicRangeProfile.fromProfile(mimeType, profile)
+
     /**
      * Get resolution according to device orientation
      *
@@ -179,6 +190,23 @@ class VideoConfig(
             format.setInteger(KEY_PRIORITY, 0) // Realtime hint
         }
 
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+            if (dynamicRangeProfile != DynamicRangeProfile.sdr) {
+                format.setInteger(
+                    MediaFormat.KEY_COLOR_STANDARD,
+                    MediaFormat.COLOR_STANDARD_BT2020
+                )
+                format.setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_FULL)
+                format.setInteger(
+                    MediaFormat.KEY_COLOR_TRANSFER,
+                    dynamicRangeProfile.transferFunction
+                )
+                format.setFeatureEnabled(
+                    MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing, true
+                )
+            }
+        }
+
         return format
     }
 
@@ -215,7 +243,6 @@ class VideoConfig(
         )
 
         private val vp9ProfilePriority = listOf(
-            VP9Profile1,
             VP9Profile0
         )
 

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt
@@ -32,8 +32,10 @@ abstract class MediaCodecEncoder<T : Config>(
     EventHandler(), IEncoder<Config> {
     protected var mediaCodec: MediaCodec? = null
         set(value) {
+            if (value != null) {
+                onNewMediaCodec(value)
+            }
             field = value
-            onNewMediaCodec()
         }
     private var callbackThread: HandlerThread? = null
     private var handler: Handler? = null
@@ -149,7 +151,7 @@ abstract class MediaCodecEncoder<T : Config>(
         }
     }
 
-    open fun onNewMediaCodec() {}
+    open fun onNewMediaCodec(mediaCodec: MediaCodec) {}
 
     open fun createMediaFormat(config: Config, withProfileLevel: Boolean) =
         config.getFormat(withProfileLevel)

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecHelper.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecHelper.kt
@@ -241,7 +241,7 @@ object MediaCodecHelper {
     fun getProfiles(
         mimeType: String,
     ): List<Int> =
-        getProfileLevel(mimeType).map { it.profile }
+        getProfileLevel(mimeType).map { it.profile }.toSet().toList()
 
     /**
      * Get encoder supported profiles list for the specified encoder.
@@ -254,7 +254,7 @@ object MediaCodecHelper {
         mimeType: String,
         name: String
     ): List<Int> =
-        getProfileLevel(mimeType, name).map { it.profile }
+        getProfileLevel(mimeType, name).map { it.profile }.toSet().toList()
 
     /**
      * Get encoder maximum supported levels for the default encoder.
@@ -321,6 +321,34 @@ object MediaCodecHelper {
         } as Boolean
     }
 
+    /**
+     * Whether the encoder supports the specified feature.
+     *
+     * @param mimeType the encoder mime type
+     * @param feature the feature to check
+     * @return true if the feature is supported, otherwise false
+     * @see MediaCodecInfo.CodecCapabilities.isFeatureSupported
+     */
+    fun isFeatureSupported(
+        mimeType: String,
+        feature: String
+    ) = getCodecCapabilities(mimeType).isFeatureSupported(feature)
+
+    /**
+     * Whether the encoder supports the specified feature.
+     *
+     * @param mimeType the encoder mime type
+     * @param name the encoder name
+     * @param feature the feature to check
+     * @return true if the feature is supported, otherwise false
+     * @see MediaCodecInfo.CodecCapabilities.isFeatureSupported
+     */
+    fun isFeatureSupported(
+        mimeType: String,
+        name: String,
+        feature: String
+    ) = getCodecCapabilities(mimeType, name).isFeatureSupported(feature)
+
     object Video {
         /**
          * Get supported video encoders list

diff --git a/...rc/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/...rc/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt
@@ -29,6 +29,7 @@ import io.github.thibaultbee.streampack.internal.gl.EglWindowSurface
 import io.github.thibaultbee.streampack.internal.gl.FullFrameRect
 import io.github.thibaultbee.streampack.internal.gl.Texture2DProgram
 import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.utils.av.video.DynamicRangeProfile
 import io.github.thibaultbee.streampack.listeners.OnErrorListener
 import java.util.concurrent.Executors
 
@@ -62,10 +63,17 @@ class VideoMediaCodecEncoder(
             _bitrate = value
         }
 
-    override fun onNewMediaCodec() {
-        mediaCodec?.let {
-            codecSurface?.outputSurface = it.createInputSurface()
+    override fun onNewMediaCodec(mediaCodec: MediaCodec) {
+        try {
+            val mimeType = mediaCodec.outputFormat.getString(MediaFormat.KEY_MIME)!!
+            val profile = mediaCodec.outputFormat.getInteger(MediaFormat.KEY_PROFILE)
+            codecSurface?.useHighBitDepth =
+                DynamicRangeProfile.fromProfile(mimeType, profile).isHdr
+        } catch (_: Exception) {
+            codecSurface?.useHighBitDepth = false
         }
+
+        codecSurface?.outputSurface = mediaCodec.createInputSurface()
     }
 
     override fun createMediaFormat(config: Config, withProfileLevel: Boolean): MediaFormat {
@@ -77,9 +85,14 @@ class VideoMediaCodecEncoder(
                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface
             )
         } else {
+            val colorFormat = if ((config as VideoConfig).dynamicRangeProfile.isHdr) {
+                MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010
+            } else {
+                MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
+            }
             videoFormat.setInteger(
                 MediaFormat.KEY_COLOR_FORMAT,
-                MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
+                colorFormat
             )
         }
         return videoFormat
@@ -124,6 +137,11 @@ class VideoMediaCodecEncoder(
         val inputSurface: Surface?
             get() = surfaceTexture?.let { Surface(surfaceTexture) }
 
+        /**
+         * If true, the encoder will use high bit depth (10 bits) for encoding.
+         */
+        var useHighBitDepth = false
+
         var outputSurface: Surface? = null
             set(value) {
                 /**
@@ -145,7 +163,7 @@ class VideoMediaCodecEncoder(
             }
 
         private fun initOrUpdateSurfaceTexture(surface: Surface) {
-            eglSurface = ensureGlContext(EglWindowSurface(surface)) {
+            eglSurface = ensureGlContext(EglWindowSurface(surface, useHighBitDepth)) {
                 val width = it.getWidth()
                 val height = it.getHeight()
                 val size =

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EglWindowSurface.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EglWindowSurface.kt
@@ -16,9 +16,14 @@
  */
 package io.github.thibaultbee.streampack.internal.gl
 
-import android.opengl.*
+import android.opengl.EGL14
+import android.opengl.EGLConfig
+import android.opengl.EGLContext
+import android.opengl.EGLDisplay
+import android.opengl.EGLExt
+import android.opengl.EGLSurface
 import android.view.Surface
-import java.util.*
+import java.util.Objects
 
 /**
  * Holds state associated with a Surface used for MediaCodec encoder input.
@@ -30,7 +35,7 @@ import java.util.*
  * (Contains mostly code borrowed from CameraX)
  */
 
-class EglWindowSurface(private val surface: Surface) {
+class EglWindowSurface(private val surface: Surface, useHighBitDepth: Boolean = false) {
     private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY
     private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT
     private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE
@@ -41,13 +46,13 @@ class EglWindowSurface(private val surface: Surface) {
     }
 
     init {
-        eglSetup()
+        eglSetup(useHighBitDepth)
     }
 
     /**
      * Prepares EGL. We want a GLES 2.0 context and a surface that supports recording.
      */
-    private fun eglSetup() {
+    private fun eglSetup(useHighBitDepth: Boolean) {
         eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
         if (Objects.equals(eglDisplay, EGL14.EGL_NO_DISPLAY)) {
             throw RuntimeException("unable to get EGL14 display")
@@ -59,12 +64,16 @@ class EglWindowSurface(private val surface: Surface) {
 
         // Configure EGL for recordable and OpenGL ES 2.0.  We want enough RGB bits
         // to minimize artifacts from possible YUV conversion.
+        val eglColorSize = if (useHighBitDepth) 10 else 8
+        val eglAlphaSize = if (useHighBitDepth) 2 else 0
+        val recordable = if (useHighBitDepth) 0 else 1
         var attribList = intArrayOf(
-            EGL14.EGL_RED_SIZE, 8,
-            EGL14.EGL_GREEN_SIZE, 8,
-            EGL14.EGL_BLUE_SIZE, 8,
+            EGL14.EGL_RED_SIZE, eglColorSize,
+            EGL14.EGL_GREEN_SIZE, eglColorSize,
+            EGL14.EGL_BLUE_SIZE, eglColorSize,
+            EGL14.EGL_ALPHA_SIZE, eglAlphaSize,
             EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
-            EGL_RECORDABLE_ANDROID, 1,
+            EGL_RECORDABLE_ANDROID, recordable,
             EGL14.EGL_NONE
         )
         val numConfigs = IntArray(1)