diff --git a/DEVELOPER_README.md b/DEVELOPER_README.md index 4cca84341..e82462aef 100644 --- a/DEVELOPER_README.md +++ b/DEVELOPER_README.md @@ -5,8 +5,8 @@ ### Definitions `Source`: -A class that represents an audio or video source. For example, a camera (`CameraCapture`), or a -microphone (`AudioCapture`). +A class that represents an audio or video source. For example, a camera (`CameraSource`), or a +microphone (`AudioSource`). `Encoder`: A class that represents an audio or video encoders. Only Android MediaCodec API is used ( @@ -76,15 +76,15 @@ Then these base streamers are specialized for a File or for a Live: There are 2 types of sources: - frames are captured in a `ByteBuffer`: such as a microphone. `ByteBuffer` sources - implement `IFrameCapture`. + implement `IFrameSource`. - frames are passed to the encoder surface (video only): when the video source can write to a `Surface`. Its purpose is to improve encoder performance. For example, it suits camera and - screen recorder. `Surface` sources implement `ISurfaceCapture`. + screen recorder. `Surface` sources implement `ISurfaceSource`. -To create a new audio source, implements a `IAudioCapture`. It inherits from `IFrameCapture`. +To create a new audio source, implements a `IAudioSource`. It inherits from `IFrameSource`. -To create a new video source, implements a `IVideoCapture`. It inherits from both `IFrameCapture` -and `ISurfaceCapture`. Always prefer to use a video source as a `Surface` source if it is possible. +To create a new video source, implements a `IVideSource`. It inherits from both `IFrameCapture` +and `ISurfaceSource`. Always prefer to use a video source as a `Surface` source if it is possible. If your video source is a `Surface` source, set: diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/streamer/BaseAudioOnlyStreamerTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/streamer/BaseAudioOnlyStreamerTest.kt index bacca947b..6e6da594c 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/streamer/BaseAudioOnlyStreamerTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/streamer/BaseAudioOnlyStreamerTest.kt @@ -33,7 +33,7 @@ class TsAudioOnlyStreamerTest : AudioOnlyStreamerTestCase() { class FlvAudioOnlyStreamerTest : AudioOnlyStreamerTestCase() { override val streamer = BaseAudioOnlyStreamer( context, - FlvMuxer(context, writeToFile = false), + FlvMuxer(writeToFile = false), FakeEndpoint(), ) } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt b/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt index 324101891..b65d44f11 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt @@ -31,12 +31,11 @@ import android.media.MediaFormat import android.os.Build import android.util.Size import io.github.thibaultbee.streampack.internal.encoders.MediaCodecHelper -import io.github.thibaultbee.streampack.internal.utils.extensions.deviceOrientation +import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait import io.github.thibaultbee.streampack.internal.utils.extensions.isVideo import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize import io.github.thibaultbee.streampack.streamers.bases.BaseStreamer -import io.github.thibaultbee.streampack.utils.OrientationUtils import java.security.InvalidParameterException import kotlin.math.roundToInt @@ -139,17 +138,8 @@ class VideoConfig( * @param context activity context * @return oriented resolution */ - fun getDeviceOrientedResolution(context: Context) = - getOrientedResolution(context.deviceOrientation) - - /** - * Get resolution according to orientation provided - * - * @param orientation the orientation - * @return oriented resolution - */ - fun getOrientedResolution(orientation: Int): Size { - return if (OrientationUtils.isPortrait(orientation)) { + fun getDeviceOrientedResolution(context: Context): Size { + return if (context.isDevicePortrait) { resolution.portraitize() } else { resolution.landscapize() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/data/orientation/DeviceOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/data/orientation/DeviceOrientationProvider.kt deleted file mode 100644 index 2dc06e194..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/data/orientation/DeviceOrientationProvider.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2023 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.internal.data.orientation - -import android.content.Context -import android.util.Size -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider -import io.github.thibaultbee.streampack.internal.utils.extensions.deviceOrientation -import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait -import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize -import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize - -class DeviceOrientationProvider(private val context: Context) : IOrientationProvider { - override val orientation: Int - get() { - //TODO: this might not be working on all devices - val deviceOrientation = context.deviceOrientation - return if (deviceOrientation == 0) 270 else deviceOrientation - 90 - } - - override fun orientedSize(size: Size): Size { - return if (context.isDevicePortrait) { - size.portraitize() - } else { - size.landscapize() - } - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/data/orientation/FixedOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/data/orientation/FixedOrientationProvider.kt deleted file mode 100644 index 8f5ee4fae..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/data/orientation/FixedOrientationProvider.kt +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2023 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.internal.data.orientation - -import android.util.Size -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider -import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize -import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize -import io.github.thibaultbee.streampack.utils.OrientationUtils - -class FixedOrientationProvider(override val orientation: Int) : IOrientationProvider { - override fun orientedSize(size: Size): Size { - return if (OrientationUtils.isPortrait(orientation)) { - size.portraitize() - } else { - size.landscapize() - } - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt index cc38e3fdf..218cedcd7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt @@ -25,10 +25,10 @@ import android.util.Size import android.view.Surface import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.data.VideoConfig -import io.github.thibaultbee.streampack.internal.gl.EGlSurface +import io.github.thibaultbee.streampack.internal.gl.EglWindowSurface import io.github.thibaultbee.streampack.internal.gl.FullFrameRect import io.github.thibaultbee.streampack.internal.gl.Texture2DProgram -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.listeners.OnErrorListener import java.util.concurrent.Executors @@ -36,12 +36,13 @@ import java.util.concurrent.Executors * Encoder for video using MediaCodec. * * @param useSurfaceMode to get video frames, if [Boolean.true],the encoder will use Surface mode, else Buffer mode with [IEncoderListener.onInputFrame]. + * @param orientationProvider to get the orientation of the source. If null, the source will keep its original dimensions. */ class VideoMediaCodecEncoder( encoderListener: IEncoderListener, override val onInternalErrorListener: OnErrorListener, private val useSurfaceMode: Boolean, - private val orientationProvider: IOrientationProvider + private val orientationProvider: ISourceOrientationProvider? ) : MediaCodecEncoder(encoderListener) { var codecSurface = if (useSurfaceMode) { @@ -63,7 +64,7 @@ class VideoMediaCodecEncoder( override fun onNewMediaCodec() { mediaCodec?.let { - codecSurface?.surface = it.createInputSurface() + codecSurface?.outputSurface = it.createInputSurface() } } @@ -86,10 +87,12 @@ class VideoMediaCodecEncoder( override fun extendMediaFormat(config: Config, format: MediaFormat) { val videoConfig = config as VideoConfig - orientationProvider.orientedSize(videoConfig.resolution).apply { - // Override previous format - format.setInteger(MediaFormat.KEY_WIDTH, width) - format.setInteger(MediaFormat.KEY_HEIGHT, height) + orientationProvider?.let { + it.getOrientedSize(videoConfig.resolution).apply { + // Override previous format + format.setInteger(MediaFormat.KEY_WIDTH, width) + format.setInteger(MediaFormat.KEY_HEIGHT, height) + } } } @@ -107,10 +110,10 @@ class VideoMediaCodecEncoder( get() = codecSurface?.inputSurface class CodecSurface( - private val orientationProvider: IOrientationProvider + private val orientationProvider: ISourceOrientationProvider? ) : SurfaceTexture.OnFrameAvailableListener { - private var eglSurface: EGlSurface? = null + private var eglSurface: EglWindowSurface? = null private var fullFrameRect: FullFrameRect? = null private var textureId = -1 private val executor = Executors.newSingleThreadExecutor() @@ -119,9 +122,8 @@ class VideoMediaCodecEncoder( val inputSurface: Surface? get() = surfaceTexture?.let { Surface(surfaceTexture) } - var surface: Surface? = null + var outputSurface: Surface? = null set(value) { - /** * When surface is called twice without the stopStream(). When configure() is * called twice for example, @@ -141,19 +143,24 @@ class VideoMediaCodecEncoder( } private fun initOrUpdateSurfaceTexture(surface: Surface) { - eglSurface = ensureGlContext(EGlSurface(surface)) { + eglSurface = ensureGlContext(EglWindowSurface(surface)) { val width = it.getWidth() val height = it.getHeight() + val size = + orientationProvider?.getOrientedSize(Size(width, height)) ?: Size(width, height) + val orientation = orientationProvider?.orientation ?: 0 fullFrameRect = FullFrameRect(Texture2DProgram()).apply { textureId = createTextureObject() setMVPMatrixAndViewPort( - orientationProvider.orientation.toFloat(), - Size(width, height) + orientation.toFloat(), + size ) } + val defaultBufferSize = + orientationProvider?.getDefaultBufferSize(size) ?: Size(width, height) surfaceTexture = attachOrBuildSurfaceTexture(surfaceTexture).apply { - setDefaultBufferSize(maxOf(height, width), minOf(height, width)) + setDefaultBufferSize(defaultBufferSize.width, defaultBufferSize.height) setOnFrameAvailableListener(this@CodecSurface) } } @@ -170,9 +177,9 @@ class VideoMediaCodecEncoder( } private fun ensureGlContext( - surface: EGlSurface?, - action: (EGlSurface) -> Unit - ): EGlSurface? { + surface: EglWindowSurface?, + action: (EglWindowSurface) -> Unit + ): EglWindowSurface? { surface?.let { it.makeCurrent() action(it) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EGlSurface.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EglWindowSurface.kt similarity index 97% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EGlSurface.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EglWindowSurface.kt index c64574fc6..319876c94 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EGlSurface.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/EglWindowSurface.kt @@ -30,15 +30,12 @@ import java.util.* * (Contains mostly code borrowed from CameraX) */ -class EGlSurface(private val surface: Surface) { +class EglWindowSurface(private val surface: Surface) { private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE private val configs = arrayOfNulls(1) - private var width = 0 - private var height = 0 - companion object { private const val EGL_RECORDABLE_ANDROID = 0x3142 } @@ -92,8 +89,6 @@ class EGlSurface(private val surface: Surface) { // Create a window surface, and attach it to the Surface we received. createEGLSurface() - width = getWidth() - height = getHeight() } private fun createEGLSurface() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/IOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/ISourceOrientationProvider.kt similarity index 60% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/IOrientationProvider.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/ISourceOrientationProvider.kt index 599d8edeb..4b2239084 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/IOrientationProvider.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/ISourceOrientationProvider.kt @@ -15,12 +15,15 @@ */ package io.github.thibaultbee.streampack.internal.interfaces +import android.graphics.SurfaceTexture import android.util.Size /** * Interface to get the orientation of the capture surface. + * These information are used to rotate the frames in the codec surface if the source needs to be rotated. + * It might not be the case for certain sources. */ -interface IOrientationProvider { +interface ISourceOrientationProvider { /** * Orientation in degrees of the surface. * Expected values: 0, 90, 180, 270. @@ -29,6 +32,15 @@ interface IOrientationProvider { /** * Return the size with the correct orientation. + * If orientation is portrait, it returns a portrait size. + * Example: + * - Size = 1920x1080, if orientation is portrait, it returns 1080x1920. */ - fun orientedSize(size: Size): Size + fun getOrientedSize(size: Size): Size + + /** + * Return the size for [SurfaceTexture.setDefaultBufferSize]. + * Override this method if the image is stretched. + */ + fun getDefaultBufferSize(size: Size) = size } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt index bbd6b954d..3ab1ee8d2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt @@ -17,13 +17,13 @@ package io.github.thibaultbee.streampack.internal.muxers import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.interfaces.Streamable -interface IMuxer: Streamable { +interface IMuxer : Streamable { val helper: IMuxerHelper - var orientationProvider: IOrientationProvider + var sourceOrientationProvider: ISourceOrientationProvider? var listener: IMuxerListener? fun encode(frame: Frame, streamPid: Int) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt index a49b6a4f4..8367b7f81 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt @@ -15,12 +15,11 @@ */ package io.github.thibaultbee.streampack.internal.muxers.flv -import android.content.Context import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame import io.github.thibaultbee.streampack.internal.data.Packet import io.github.thibaultbee.streampack.internal.data.PacketType -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.muxers.flv.tags.AVTagsFactory @@ -31,7 +30,6 @@ import io.github.thibaultbee.streampack.internal.utils.extensions.isAudio import io.github.thibaultbee.streampack.internal.utils.extensions.isVideo class FlvMuxer( - private val context: Context, override var listener: IMuxerListener? = null, initialStreams: List? = null, private val writeToFile: Boolean, @@ -49,7 +47,7 @@ class FlvMuxer( initialStreams?.let { streams.addAll(it) } } - override lateinit var orientationProvider: IOrientationProvider + override var sourceOrientationProvider: ISourceOrientationProvider? = null override fun encode(frame: Frame, streamPid: Int) { if (!hasFirstFrame) { @@ -60,6 +58,7 @@ class FlvMuxer( startUpTime = frame.pts hasFirstFrame = true } else { + // Drop return } } else { @@ -113,7 +112,7 @@ class FlvMuxer( // Metadata listener?.onOutputFrame( Packet( - OnMetadata(orientationProvider, streams).write(), + OnMetadata.fromConfigs(streams, sourceOrientationProvider).write(), TimeUtils.currentTime() ) ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt index e1d670273..b49cf8815 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.internal.muxers.flv.tags import io.github.thibaultbee.streampack.data.AudioConfig import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.data.VideoConfig -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.flv.amf.containers.AmfContainer import io.github.thibaultbee.streampack.internal.muxers.flv.amf.containers.AmfEcmaArray import io.github.thibaultbee.streampack.internal.muxers.flv.tags.video.CodecID @@ -31,8 +31,8 @@ import java.io.IOException import java.nio.ByteBuffer class OnMetadata( - orientationProvider: IOrientationProvider, - streams: List + streams: List, + duration: Double = 0.0 ) : FlvTag(0, TagType.SCRIPT) { private val amfContainer = AmfContainer() @@ -40,53 +40,38 @@ class OnMetadata( init { amfContainer.add("onMetaData") val ecmaArray = AmfEcmaArray() - ecmaArray.add("duration", 0.0) + ecmaArray.add("duration", duration) streams.forEach { when (it) { - is AudioConfig -> { + is AudioMetadata -> { ecmaArray.add( "audiocodecid", - SoundFormat.fromMimeType(it.mimeType).value.toDouble() + it.codecID.value.toDouble() ) - ecmaArray.add("audiodatarate", it.startBitrate.toDouble() / 1000) // to Kpbs + ecmaArray.add("audiodatarate", it.dataRate.toDouble() / 1000) // to Kbps ecmaArray.add("audiosamplerate", it.sampleRate.toDouble()) ecmaArray.add( "audiosamplesize", - it.byteFormat.numOfBits().toDouble() + it.sampleSize.toDouble() ) ecmaArray.add( "stereo", - AudioConfig.getNumberOfChannels(it.channelConfig) == 2 + it.isStereo ) } - is VideoConfig -> { - val resolution = orientationProvider.orientedSize(it.resolution) - val videoCodecID = if (ExtendedVideoTag.isSupportedCodec(it.mimeType)) { - FourCCs.fromMimeType(it.mimeType).value.code - } else { - try { - CodecID.fromMimeType(it.mimeType).value - } catch (e: Exception) { - Logger.e(TAG, "Failed to get videocodecid for: ${it.mimeType}") - null - } - } - videoCodecID?.let { codecId -> + is VideoMetadata -> { + it.codecID?.let { codecId -> ecmaArray.add( "videocodecid", codecId.toDouble() ) } - ecmaArray.add("videodatarate", it.startBitrate.toDouble() / 1000) // to Kpbs - ecmaArray.add("width", resolution.width.toDouble()) - ecmaArray.add("height", resolution.height.toDouble()) - ecmaArray.add("framerate", it.fps.toDouble()) - } - - else -> { - throw IOException("Not supported mime type: ${it.mimeType}") + ecmaArray.add("videodatarate", it.dataRate.toDouble() / 1000) // to Kbps + ecmaArray.add("width", it.width.toDouble()) + ecmaArray.add("height", it.height.toDouble()) + ecmaArray.add("framerate", it.frameRate.toDouble()) } } } @@ -106,4 +91,83 @@ class OnMetadata( override val bodySize: Int get() = amfContainer.size + + companion object { + fun fromConfigs( + configs: List, + sourceOrientationProvider: ISourceOrientationProvider? = null + ): OnMetadata { + return OnMetadata(configs.map { Metadata.fromConfig(it, sourceOrientationProvider) }) + } + } +} + +abstract class Metadata(val dataRate: Int) { + companion object { + fun fromConfig( + config: Config, + sourceOrientationProvider: ISourceOrientationProvider? = null + ): Metadata { + return when (config) { + is AudioConfig -> AudioMetadata.fromAudioConfig(config) + is VideoConfig -> VideoMetadata.fromVideoConfig(config, sourceOrientationProvider) + else -> throw IOException("Not supported mime type: ${config.mimeType}") + } + } + } +} + +class AudioMetadata( + val codecID: SoundFormat, + dataRate: Int, + val sampleRate: Int, + val sampleSize: Int, + val isStereo: Boolean +) : Metadata(dataRate) { + companion object { + fun fromAudioConfig(config: AudioConfig): AudioMetadata { + return AudioMetadata( + SoundFormat.fromMimeType(config.mimeType), + config.startBitrate, + config.sampleRate, + config.byteFormat.numOfBits(), + AudioConfig.getNumberOfChannels(config.channelConfig) == 2 + ) + } + } +} + +class VideoMetadata( + val codecID: Int?, + dataRate: Int, + val width: Int, + val height: Int, + val frameRate: Int +) : Metadata(dataRate) { + companion object { + fun fromVideoConfig( + config: VideoConfig, + sourceOrientationProvider: ISourceOrientationProvider? = null + ): VideoMetadata { + val videoCodecID = if (ExtendedVideoTag.isSupportedCodec(config.mimeType)) { + FourCCs.fromMimeType(config.mimeType).value.code + } else { + try { + CodecID.fromMimeType(config.mimeType).value + } catch (e: Exception) { + Logger.e(TAG, "Failed to get videocodecid for: ${config.mimeType}") + null + } + } + val resolution = + sourceOrientationProvider?.getOrientedSize(config.resolution) ?: config.resolution + return VideoMetadata( + videoCodecID, + config.startBitrate, + resolution.width, + resolution.height, + config.fps + ) + } + } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt index d769597aa..de905c680 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.internal.muxers.mp4 import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame import io.github.thibaultbee.streampack.internal.data.Packet -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.FileTypeBox @@ -43,7 +43,7 @@ class MP4Muxer( private val segmenterFactory: MP4SegmenterFactory = DefaultMP4SegmenterFactory() ) : IMuxer { override val helper = MP4MuxerHelper() - override lateinit var orientationProvider: IOrientationProvider + override var sourceOrientationProvider: ISourceOrientationProvider? = null override var listener: IMuxerListener? = initialListener private val tracks = mutableListOf() @@ -116,8 +116,7 @@ class MP4Muxer( private fun createNewSegment(movieBoxFactory: AbstractMovieBoxFactory): Segment { return Segment( tracks, - movieBoxFactory, - orientationProvider, + movieBoxFactory ) { buffer -> writeBuffer(buffer) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/Segment.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/Segment.kt index 7b8bdfe43..6097f6f97 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/Segment.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/Segment.kt @@ -16,7 +16,6 @@ package io.github.thibaultbee.streampack.internal.muxers.mp4.models import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.MediaDataBox import java.nio.ByteBuffer @@ -26,7 +25,6 @@ import java.nio.ByteBuffer class Segment( tracks: List, private val movieBoxFactory: AbstractMovieBoxFactory, - private val orientationProvider: IOrientationProvider, private val onNewSample: (ByteBuffer) -> Unit ) { /** @@ -34,7 +32,7 @@ class Segment( */ val isFragment = movieBoxFactory is MovieFragmentBoxFactory - private val trackChunks = tracks.map { TrackChunks(it, orientationProvider, onNewSample) } + private val trackChunks = tracks.map { TrackChunks(it, onNewSample) } private val validTrackChunks: List get() = trackChunks.filter { it.isValid } private val validDataSize: Int diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/TrackChunks.kt index 1913b292e..b1fc0fd5c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/TrackChunks.kt @@ -21,7 +21,6 @@ import io.github.thibaultbee.streampack.data.AudioConfig import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AV1CodecConfigurationBox2 import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AV1SampleEntry import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AVCConfigurationBox @@ -68,6 +67,7 @@ import io.github.thibaultbee.streampack.internal.utils.extensions.clone import io.github.thibaultbee.streampack.internal.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.internal.utils.extensions.isAvcc import io.github.thibaultbee.streampack.internal.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.internal.utils.extensions.resolution import io.github.thibaultbee.streampack.internal.utils.extensions.startCodeSize import java.nio.ByteBuffer @@ -78,7 +78,6 @@ import java.nio.ByteBuffer */ class TrackChunks( val track: Track, - private val orientationProvider: IOrientationProvider, val onNewSample: (ByteBuffer) -> Unit, ) { private val chunks = mutableListOf() @@ -218,7 +217,7 @@ class TrackChunks( } fun createTrak(firstChunkOffset: Long): TrackBox { - val tkhd = createTrackHeaderBox(track.config) + val tkhd = createTrackHeaderBox(track.config, format.first()) val mdhd = MediaHeaderBox(version = 0, timescale = track.timescale, duration = duration) val hdlr = track.config.createHandlerBox() @@ -237,10 +236,10 @@ class TrackChunks( return TrackBox(tkhd, mdia) } - private fun createTrackHeaderBox(config: Config): TrackHeaderBox { + private fun createTrackHeaderBox(config: Config, format: MediaFormat): TrackHeaderBox { val resolution = when (config) { is AudioConfig -> Size(0, 0) - is VideoConfig -> orientationProvider.orientedSize(config.resolution) + is VideoConfig -> format.resolution else -> throw IllegalArgumentException("Unsupported config") } val volume = when (config) { @@ -310,11 +309,12 @@ class TrackChunks( private fun createSampleDescriptionBox(): SampleDescriptionBox { val sampleEntry = when (track.config.mimeType) { MediaFormat.MIMETYPE_VIDEO_AVC -> { + val format = this.format.first() val extra = this.extra require(extra.size == 2) { "For AVC, extra must contain 2 parameter sets" } (track.config as VideoConfig) AVCSampleEntry( - orientationProvider.orientedSize(track.config.resolution), + format.resolution, avcC = AVCConfigurationBox( AVCDecoderConfigurationRecord.fromParameterSets( extra[0], @@ -325,11 +325,12 @@ class TrackChunks( } MediaFormat.MIMETYPE_VIDEO_HEVC -> { + val format = this.format.first() val extra = this.extra require(extra.size == 3) { "For HEVC, extra must contain 3 parameter sets" } (track.config as VideoConfig) HEVCSampleEntry( - orientationProvider.orientedSize(track.config.resolution), + format.resolution, hvcC = HEVCConfigurationBox( HEVCDecoderConfigurationRecord.fromParameterSets( extra.flatten() @@ -339,22 +340,23 @@ class TrackChunks( } MediaFormat.MIMETYPE_VIDEO_VP9 -> { - val format = this.format + val format = this.format.first() (track.config as VideoConfig) VP9SampleEntry( - orientationProvider.orientedSize(track.config.resolution), + format.resolution, vpcC = VPCodecConfigurationBox( - VPCodecConfigurationRecord.fromMediaFormat(format.first()) + VPCodecConfigurationRecord.fromMediaFormat(format) ), ) } MediaFormat.MIMETYPE_VIDEO_AV1 -> { + val format = this.format.first() val extra = this.extra require(extra.size == 1) { "For AV1, extra must contain 1 extra" } (track.config as VideoConfig) AV1SampleEntry( - orientationProvider.orientedSize(track.config.resolution), + format.resolution, av1C = AV1CodecConfigurationBox2( extra[0][0] ), diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt index 0ec416ffe..84a18a2ec 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt @@ -20,7 +20,7 @@ import android.media.MediaFormat import io.github.thibaultbee.streampack.data.AudioConfig import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.muxers.ts.data.Service @@ -46,7 +46,7 @@ class TSMuxer( override val helper = TSMuxerHelper() private val tsServices = mutableListOf() private val tsPes = mutableListOf() - override lateinit var orientationProvider: IOrientationProvider + override var sourceOrientationProvider: ISourceOrientationProvider? = null override var listener: IMuxerListener? = initialListener set(value) { pat.listener = value diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/AudioCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/AudioSource.kt similarity index 95% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/AudioCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/AudioSource.kt index 7ba8848a2..5676b1059 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/AudioCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/AudioSource.kt @@ -31,7 +31,7 @@ import io.github.thibaultbee.streampack.logger.Logger import io.github.thibaultbee.streampack.utils.TAG import java.nio.ByteBuffer -class AudioCapture : IAudioCapture { +class AudioSource : IAudioSource { private var audioRecord: AudioRecord? = null private var mutedByteArray: ByteArray? = null @@ -85,9 +85,9 @@ class AudioCapture : IAudioCapture { it.startRecording() if (!isRunning()) { - throw IllegalStateException("AudioCapture: failed to start recording") + throw IllegalStateException("AudioSource: failed to start recording") } - } ?: throw IllegalStateException("AudioCapture: run: : No audioRecorder") + } ?: throw IllegalStateException("AudioSource: run: : No audioRecorder") } private fun isRunning() = audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING @@ -148,7 +148,7 @@ class AudioCapture : IAudioCapture { } else { throw IllegalArgumentException(audioRecordErrorToString(length)) } - } ?: throw IllegalStateException("AudioCapture: getFrame: No audioRecorder") + } ?: throw IllegalStateException("AudioSource: getFrame: No audioRecorder") } private fun audioRecordErrorToString(audioRecordError: Int) = when (audioRecordError) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IAudioCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IAudioSource.kt similarity index 85% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IAudioCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IAudioSource.kt index 52af4e5d3..ba19a64d5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IAudioCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IAudioSource.kt @@ -17,9 +17,9 @@ package io.github.thibaultbee.streampack.internal.sources import io.github.thibaultbee.streampack.data.AudioConfig -interface IAudioCapture : IFrameCapture { +interface IAudioSource : IFrameSource { /** - * [Boolean.true] to mute [IAudioCapture], [Boolean.false] to unmute. + * [Boolean.true] to mute [IAudioSource], [Boolean.false] to unmute. */ var isMuted: Boolean } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IFrameCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IFrameSource.kt similarity index 96% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IFrameCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IFrameSource.kt index 5daf1ace8..e1bc889a3 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IFrameCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IFrameSource.kt @@ -19,7 +19,7 @@ import io.github.thibaultbee.streampack.internal.data.Frame import io.github.thibaultbee.streampack.internal.interfaces.Streamable import java.nio.ByteBuffer -interface IFrameCapture : Streamable { +interface IFrameSource : Streamable { /** * Generate a frame from capture device diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/ISurfaceCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/ISurfaceSource.kt similarity index 95% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/ISurfaceCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/ISurfaceSource.kt index 227f9ffa2..d6052ef64 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/ISurfaceCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/ISurfaceSource.kt @@ -19,7 +19,7 @@ import android.view.Surface import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.internal.interfaces.Streamable -interface ISurfaceCapture : Streamable { +interface ISurfaceSource : Streamable { /** * The offset between source capture time and MONOTONIC clock. It is used to synchronize video * with audio. It is only useful for camera source. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt similarity index 60% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt index 86c41f6f9..393d7f4b7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt @@ -16,12 +16,24 @@ package io.github.thibaultbee.streampack.internal.sources import io.github.thibaultbee.streampack.data.VideoConfig +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider -interface IVideoCapture : IFrameCapture, ISurfaceCapture { +interface IVideoSource : IFrameSource, ISurfaceSource { /** - * Set to [Boolean.true] to use video source as a Surface renderer (see [ISurfaceCapture]). For example, this is useful + * Set to [Boolean.true] to use video source as a Surface renderer (see [ISurfaceSource]). For example, this is useful * for camera and screen recording. If set to [Boolean.false], the encoder will use source as a - * buffer producer (see [IFrameCapture]). + * buffer producer (see [IFrameSource]). */ val hasSurface: Boolean + + /** + * Set to [Boolean.true] to use video source as a buffer producer (see [IFrameSource]). + */ + val hasFrames: Boolean + + /** + * Orientation provider of the capture source. + * It is used to orientate the frame according to the source orientation. + */ + val orientationProvider: ISourceOrientationProvider } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index 2cde9e6c9..667d76064 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -27,6 +27,7 @@ import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.error.CameraError import io.github.thibaultbee.streampack.logger.Logger import io.github.thibaultbee.streampack.utils.TAG +import io.github.thibaultbee.streampack.utils.getCameraFpsList import kotlinx.coroutines.* import java.security.InvalidParameterException import kotlin.coroutines.resume diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt similarity index 71% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt index 216ad496e..b56baff3b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt @@ -17,22 +17,31 @@ package io.github.thibaultbee.streampack.internal.sources.camera import android.Manifest import android.content.Context +import android.util.Size import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.sources.IVideoCapture +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.sources.IVideoSource +import io.github.thibaultbee.streampack.internal.utils.extensions.deviceOrientation +import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait +import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize +import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize import io.github.thibaultbee.streampack.utils.CameraSettings import io.github.thibaultbee.streampack.utils.defaultCameraId import io.github.thibaultbee.streampack.utils.isFrameRateSupported import kotlinx.coroutines.runBlocking import java.nio.ByteBuffer +import kotlin.math.max +import kotlin.math.min -class CameraCapture( +class CameraSource( private val context: Context, -) : IVideoCapture { +) : IVideoSource { var previewSurface: Surface? = null override var encoderSurface: Surface? = null + var cameraId: String = context.defaultCameraId get() = cameraController.cameraId ?: field @RequiresPermission(Manifest.permission.CAMERA) @@ -51,10 +60,13 @@ class CameraCapture( } } private var cameraController = CameraController(context) - var settings = CameraSettings(context, cameraController) + val settings = CameraSettings(context, cameraController) override val timestampOffset = CameraHelper.getTimeOffsetToMonoClock(context, cameraId) override val hasSurface = true + override val hasFrames = false + override val orientationProvider = CameraOrientationProvider(context) + override fun getFrame(buffer: ByteBuffer): Frame { throw UnsupportedOperationException("Camera expects to run in Surface mode") } @@ -113,4 +125,30 @@ class CameraCapture( override fun release() { cameraController.release() } + + + class CameraOrientationProvider(private val context: Context) : + ISourceOrientationProvider { + + override val orientation: Int + get() = when (context.deviceOrientation) { + Surface.ROTATION_0 -> 0 + Surface.ROTATION_90 -> 270 + Surface.ROTATION_180 -> 180 + Surface.ROTATION_270 -> 90 + else -> 0 + } + + override fun getOrientedSize(size: Size): Size { + return if (context.isDevicePortrait) { + size.portraitize() + } else { + size.landscapize() + } + } + + override fun getDefaultBufferSize(size: Size): Size { + return Size(max(size.width, size.height), min(size.width, size.height)) + } + } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/InternalCameraExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/InternalCameraExtensions.kt deleted file mode 100644 index 5694246d1..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/InternalCameraExtensions.kt +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2021 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.internal.sources.camera - -import android.content.Context -import android.graphics.ImageFormat -import android.hardware.camera2.CameraCharacteristics -import android.util.Range -import android.util.Size -import io.github.thibaultbee.streampack.utils.cameraList -import io.github.thibaultbee.streampack.utils.getCameraCharacteristics - -/** - * Gets all output capture sizes. - * - * @return List of resolutions supported by all camera - */ -fun Context.getCameraOutputStreamSizes(): List { - val cameraIdList = cameraList - val resolutionSet = mutableSetOf() - cameraIdList.forEach { cameraId -> - resolutionSet.addAll(getCameraOutputStreamSizes(cameraId)) - } - return resolutionSet.toList() -} - -/** - * Gets list of output stream sizes of a camera. - * - * @param cameraId camera id - * @return List of resolutions supported by a camera - * @see [Context.getCameraOutputStreamSizes] - */ -fun Context.getCameraOutputStreamSizes(cameraId: String): List { - return this.getCameraCharacteristics(cameraId)[CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP]?.getOutputSizes( - ImageFormat.YUV_420_888 - )?.toList() ?: emptyList() -} - -/** - * Get list of framerate for a camera. - * - * @param cameraId camera id - * @return List of fps supported by a camera - */ -fun Context.getCameraFpsList(cameraId: String): List> { - return this.getCameraCharacteristics(cameraId)[CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES]?.toList() - ?: emptyList() -} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenCapture.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt similarity index 73% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenCapture.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt index 6369968c4..bfb48b69f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenCapture.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt @@ -22,23 +22,31 @@ import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.Handler import android.os.HandlerThread +import android.util.Size import android.view.Surface import androidx.activity.result.ActivityResult import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.error.StreamPackError import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.sources.IVideoCapture +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.sources.IVideoSource +import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait +import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize +import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize import io.github.thibaultbee.streampack.listeners.OnErrorListener import io.github.thibaultbee.streampack.logger.Logger import io.github.thibaultbee.streampack.utils.TAG import java.nio.ByteBuffer -class ScreenCapture( +class ScreenSource( context: Context -) : IVideoCapture { +) : IVideoSource { override var encoderSurface: Surface? = null override val timestampOffset = 0L override val hasSurface = true + override val hasFrames = false + override val orientationProvider = ScreenSourceOrientationProvider(context) + override fun getFrame(buffer: ByteBuffer): Frame { throw UnsupportedOperationException("Screen source expects to run in Surface mode") } @@ -56,26 +64,26 @@ class ScreenCapture( private val virtualDisplayCallback = object : VirtualDisplay.Callback() { override fun onPaused() { super.onPaused() - Logger.i(this@ScreenCapture.TAG, "onPaused") + Logger.i(this@ScreenSource.TAG, "onPaused") } override fun onStopped() { super.onStopped() - Logger.i(this@ScreenCapture.TAG, "onStopped") - onErrorListener?.onError(StreamPackError("Screen capture has been stopped")) + Logger.i(this@ScreenSource.TAG, "onStopped") + onErrorListener?.onError(StreamPackError("Screen source has been stopped")) } } private val mediaProjectionCallback = object : MediaProjection.Callback() { override fun onStop() { super.onStop() - Logger.i(this@ScreenCapture.TAG, "onStop") - onErrorListener?.onError(StreamPackError("Screen capture has been stopped")) + Logger.i(this@ScreenSource.TAG, "onStop") + onErrorListener?.onError(StreamPackError("Screen source has been stopped")) } } companion object { - private const val VIRTUAL_DISPLAY_NAME = "StreamPackScreenCapture" + private const val VIRTUAL_DISPLAY_NAME = "StreamPackScreenSource" } override fun configure(config: VideoConfig) { @@ -89,11 +97,12 @@ class ScreenCapture( val resultCode = activityResult!!.resultCode val resultData = activityResult!!.data!! + val orientedSize = orientationProvider.getOrientedSize(videoConfig!!.resolution) mediaProjection = mediaProjectionManager.getMediaProjection(resultCode, resultData).apply { virtualDisplay = createVirtualDisplay( VIRTUAL_DISPLAY_NAME, - videoConfig!!.resolution.width, - videoConfig!!.resolution.height, + orientedSize.width, + orientedSize.height, 320, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, encoderSurface, @@ -120,4 +129,17 @@ class ScreenCapture( e.printStackTrace() } } + + class ScreenSourceOrientationProvider(private val context: Context) : + ISourceOrientationProvider { + override val orientation = 0 + + override fun getOrientedSize(size: Size): Size { + return if (context.isDevicePortrait) { + size.portraitize() + } else { + size.landscapize() + } + } + } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/ContextExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/ContextExtensions.kt index 11f070150..4cbc4325b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/ContextExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/ContextExtensions.kt @@ -16,6 +16,8 @@ package io.github.thibaultbee.streampack.internal.utils.extensions import android.content.Context +import android.content.res.Configuration.ORIENTATION_LANDSCAPE +import android.content.res.Configuration.ORIENTATION_PORTRAIT import android.hardware.display.DisplayManager import android.view.Display import android.view.Surface @@ -26,21 +28,23 @@ import io.github.thibaultbee.streampack.utils.OrientationUtils /** * Returns the device orientation in degrees from the natural orientation: portrait. * - * @return the device orientation + * @return the device orientation in degrees + */ +val Context.deviceOrientationDegrees: Int + get() { + val displayManager = this.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager + return OrientationUtils.getSurfaceOrientationDegrees(displayManager.getDisplay(Display.DEFAULT_DISPLAY).rotation) + } + +/** + * Returns the device orientation in degrees from the natural orientation: portrait. + * + * @return the device orientation as [Surface.ROTATION_0], [Surface.ROTATION_90], [Surface.ROTATION_180] or [Surface.ROTATION_270] */ val Context.deviceOrientation: Int get() { val displayManager = this.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager - return when (val displayRotation = - displayManager.getDisplay(Display.DEFAULT_DISPLAY).rotation) { - Surface.ROTATION_0 -> 90 - Surface.ROTATION_90 -> 0 - Surface.ROTATION_180 -> 270 - Surface.ROTATION_270 -> 180 - else -> throw UnsupportedOperationException( - "Unsupported display rotation: $displayRotation" - ) - } + return displayManager.getDisplay(Display.DEFAULT_DISPLAY).rotation } /** @@ -49,10 +53,8 @@ val Context.deviceOrientation: Int * @return true if the device is in portrait, otherwise false */ val Context.isDevicePortrait: Boolean - get() { - val orientation = this.deviceOrientation - return OrientationUtils.isPortrait(orientation) - } + get() = resources.configuration.orientation == ORIENTATION_PORTRAIT + /** * Check if the device is in landscape. @@ -60,10 +62,7 @@ val Context.isDevicePortrait: Boolean * @return true if the device is in landscape, otherwise false */ val Context.isDeviceLandscape: Boolean - get() = !isDevicePortrait - -val Context.naturalOrientation: Int - get() = resources.configuration.orientation + get() = resources.configuration.orientation == ORIENTATION_LANDSCAPE val Context.defaultTsServiceInfo get() = TsServiceInfo( diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/Extensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/Extensions.kt index a66ce7098..a8fdbbba6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/Extensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/Extensions.kt @@ -22,15 +22,23 @@ import android.util.Rational fun Any.numOfBits(): Int { return when (this) { - is Byte -> 8 - is Short -> 16 - is Int -> 32 - is Long -> 64 - is Float -> 32 - is Double -> 64 + is Byte -> Byte.SIZE_BITS + is Short -> Short.SIZE_BITS + is Int -> Int.SIZE_BITS + is Long -> Long.SIZE_BITS + is Float -> Float.SIZE_BITS + is Double -> Double.SIZE_BITS is Boolean -> 1 - is Char -> 16 - is String -> 8 * this.length + is Char -> Char.SIZE_BITS + is String -> Char.SIZE_BITS * length + is ByteArray -> Byte.SIZE_BITS * size + is ShortArray -> Short.SIZE_BITS * size + is IntArray -> Int.SIZE_BITS * size + is LongArray -> Long.SIZE_BITS * size + is FloatArray -> Float.SIZE_BITS * size + is DoubleArray -> Double.SIZE_BITS * size + is BooleanArray -> size + is CharArray -> Char.SIZE_BITS * size else -> throw IllegalArgumentException("Unsupported type: ${this.javaClass.name}") } } @@ -68,9 +76,3 @@ fun PointF.normalize(rect: Rect): PointF { } fun Rational.flip() = Rational(denominator, numerator) - -val Int.isOrientationPortrait: Boolean - get() = this == 90 || this == 270 - -val Int.isOrientationLandscape: Boolean - get() = this == 0 || this == 180 diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/MediaFormatExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/MediaFormatExtensions.kt index d1cc66dfc..96f0bfdfa 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/MediaFormatExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/MediaFormatExtensions.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.internal.utils.extensions import android.media.MediaFormat +import android.util.Size import java.nio.ByteBuffer /** @@ -48,4 +49,16 @@ val MediaFormat.extra: List } return extra + } + +/** + * Extracts resolution from a [MediaFormat]. + * Only for [MediaFormat] of video. + */ +val MediaFormat.resolution: Size + get() { + return Size( + getInteger(MediaFormat.KEY_WIDTH), + getInteger(MediaFormat.KEY_HEIGHT) + ) } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseAudioOnlyStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseAudioOnlyStreamer.kt index 6a1447d9b..3b09a9483 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseAudioOnlyStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseAudioOnlyStreamer.kt @@ -16,10 +16,9 @@ package io.github.thibaultbee.streampack.streamers.bases import android.content.Context -import io.github.thibaultbee.streampack.internal.data.orientation.FixedOrientationProvider import io.github.thibaultbee.streampack.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.internal.muxers.IMuxer -import io.github.thibaultbee.streampack.internal.sources.AudioCapture +import io.github.thibaultbee.streampack.internal.sources.AudioSource import io.github.thibaultbee.streampack.listeners.OnErrorListener /** @@ -37,9 +36,8 @@ open class BaseAudioOnlyStreamer( initialOnErrorListener: OnErrorListener? = null ) : BaseStreamer( context = context, - videoCapture = null, - audioCapture = AudioCapture(), - orientationProvider = FixedOrientationProvider(orientation = 0), // Unused + videoSource = null, + audioSource = AudioSource(), muxer = muxer, endpoint = endpoint, initialOnErrorListener = initialOnErrorListener diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseCameraStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseCameraStreamer.kt index d00995b3e..c5c8843a6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseCameraStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseCameraStreamer.kt @@ -22,16 +22,16 @@ import android.view.SurfaceView import android.view.TextureView import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.error.StreamPackError -import io.github.thibaultbee.streampack.internal.data.orientation.DeviceOrientationProvider import io.github.thibaultbee.streampack.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.internal.muxers.IMuxer -import io.github.thibaultbee.streampack.internal.sources.AudioCapture -import io.github.thibaultbee.streampack.internal.sources.camera.CameraCapture +import io.github.thibaultbee.streampack.internal.sources.AudioSource +import io.github.thibaultbee.streampack.internal.sources.camera.CameraSource import io.github.thibaultbee.streampack.listeners.OnErrorListener import io.github.thibaultbee.streampack.streamers.helpers.CameraStreamerConfigurationHelper import io.github.thibaultbee.streampack.streamers.interfaces.ICameraStreamer import io.github.thibaultbee.streampack.streamers.settings.BaseCameraStreamerSettings import io.github.thibaultbee.streampack.views.AutoFitSurfaceView +import io.github.thibaultbee.streampack.views.PreviewView import kotlinx.coroutines.runBlocking /** @@ -51,14 +51,13 @@ open class BaseCameraStreamer( initialOnErrorListener: OnErrorListener? = null ) : BaseStreamer( context = context, - videoCapture = CameraCapture(context), - audioCapture = if (enableAudio) AudioCapture() else null, - orientationProvider = DeviceOrientationProvider(context), + videoSource = CameraSource(context), + audioSource = if (enableAudio) AudioSource() else null, muxer = muxer, endpoint = endpoint, initialOnErrorListener = initialOnErrorListener ), ICameraStreamer { - private val cameraCapture = videoCapture as CameraCapture + private val cameraSource = videoSource as CameraSource override val helper = CameraStreamerConfigurationHelper(muxer.helper) /** @@ -70,7 +69,7 @@ open class BaseCameraStreamer( * * @return a string that described current camera */ - get() = cameraCapture.cameraId + get() = cameraSource.cameraId /** * Set current camera id. * @@ -78,11 +77,11 @@ open class BaseCameraStreamer( */ @RequiresPermission(Manifest.permission.CAMERA) set(value) { - cameraCapture.cameraId = value + cameraSource.cameraId = value } override var settings = - BaseCameraStreamerSettings(audioCapture, cameraCapture, audioEncoder, videoEncoder) + BaseCameraStreamerSettings(audioSource, cameraSource, audioEncoder, videoEncoder) /** * Starts audio and video capture. @@ -90,7 +89,7 @@ open class BaseCameraStreamer( * * Inside, it launches both camera and microphone capture. * - * @param previewSurface Where to display camera capture. Could be a [Surface] from a [StreamerSurfaceView], an [AutoFitSurfaceView], a [SurfaceView] or a [TextureView]. + * @param previewSurface Where to display camera capture. Could be a [Surface] from a [PreviewView], an [AutoFitSurfaceView], a [SurfaceView] or a [TextureView]. * @param cameraId camera id (get camera id list from [Context.cameraList]) * * @throws [StreamPackError] if audio or video capture couldn't be launch @@ -101,9 +100,9 @@ open class BaseCameraStreamer( require(videoConfig != null) { "Video has not been configured!" } runBlocking { try { - cameraCapture.previewSurface = previewSurface - cameraCapture.encoderSurface = videoEncoder?.inputSurface - cameraCapture.startPreview(cameraId) + cameraSource.previewSurface = previewSurface + cameraSource.encoderSurface = videoEncoder?.inputSurface + cameraSource.startPreview(cameraId) } catch (e: Exception) { stopPreview() throw StreamPackError(e) @@ -119,7 +118,7 @@ open class BaseCameraStreamer( */ override fun stopPreview() { stopStream() - cameraCapture.stopPreview() + cameraSource.stopPreview() } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseScreenRecorderStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseScreenRecorderStreamer.kt index 2a763f5c8..d742b64f4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseScreenRecorderStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseScreenRecorderStreamer.kt @@ -21,11 +21,10 @@ import android.media.projection.MediaProjectionManager import androidx.activity.ComponentActivity import androidx.activity.result.ActivityResult import androidx.core.app.ActivityCompat -import io.github.thibaultbee.streampack.internal.data.orientation.FixedOrientationProvider import io.github.thibaultbee.streampack.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.internal.muxers.IMuxer -import io.github.thibaultbee.streampack.internal.sources.AudioCapture -import io.github.thibaultbee.streampack.internal.sources.screen.ScreenCapture +import io.github.thibaultbee.streampack.internal.sources.AudioSource +import io.github.thibaultbee.streampack.internal.sources.screen.ScreenSource import io.github.thibaultbee.streampack.listeners.OnErrorListener /** @@ -45,15 +44,14 @@ open class BaseScreenRecorderStreamer( initialOnErrorListener: OnErrorListener? = null ) : BaseStreamer( context = context, - videoCapture = ScreenCapture(context), - audioCapture = if (enableAudio) AudioCapture() else null, - orientationProvider = FixedOrientationProvider(orientation = 0), + videoSource = ScreenSource(context), + audioSource = if (enableAudio) AudioSource() else null, muxer = muxer, endpoint = endpoint, initialOnErrorListener = initialOnErrorListener ) { - private val screenCapture = - (videoCapture as ScreenCapture).apply { onErrorListener = onInternalErrorListener } + private val screenSource = + (videoSource as ScreenSource).apply { onErrorListener = onInternalErrorListener } companion object { /** @@ -79,22 +77,22 @@ open class BaseScreenRecorderStreamer( * * @return activity result previously set. */ - get() = screenCapture.activityResult + get() = screenSource.activityResult /** * Set activity result. Must be call before [startStream]. * * @param value activity result returns from [ComponentActivity.registerForActivityResult] callback. */ set(value) { - screenCapture.activityResult = value + screenSource.activityResult = value } /** - * Same as [BaseStreamer] but it prepares [ScreenCapture.encoderSurface]. + * Same as [BaseStreamer] but it prepares [ScreenSource.encoderSurface]. * You must have set [activityResult] before. */ override fun startStream() { - screenCapture.encoderSurface = videoEncoder?.inputSurface + screenSource.encoderSurface = videoEncoder?.inputSurface super.startStream() } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt index 2600e63cb..21757714b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt @@ -29,11 +29,11 @@ import io.github.thibaultbee.streampack.internal.encoders.IEncoderListener import io.github.thibaultbee.streampack.internal.encoders.VideoMediaCodecEncoder import io.github.thibaultbee.streampack.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.internal.events.EventHandler -import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider +import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener -import io.github.thibaultbee.streampack.internal.sources.IAudioCapture -import io.github.thibaultbee.streampack.internal.sources.IVideoCapture +import io.github.thibaultbee.streampack.internal.sources.IAudioSource +import io.github.thibaultbee.streampack.internal.sources.IVideoSource import io.github.thibaultbee.streampack.listeners.OnErrorListener import io.github.thibaultbee.streampack.logger.Logger import io.github.thibaultbee.streampack.streamers.helpers.IConfigurationHelper @@ -48,18 +48,16 @@ import java.nio.ByteBuffer * Base class of all streamers. * * @param context application context - * @param videoCapture Video source - * @param audioCapture Audio source - * @param orientationProvider The orientation provider + * @param videoSource Video source + * @param audioSource Audio source * @param muxer a [IMuxer] implementation * @param endpoint a [IEndpoint] implementation * @param initialOnErrorListener initialize [OnErrorListener] */ abstract class BaseStreamer( private val context: Context, - protected val audioCapture: IAudioCapture?, - protected val videoCapture: IVideoCapture?, - orientationProvider: IOrientationProvider, + protected val audioSource: IAudioSource?, + protected val videoSource: IVideoSource?, private val muxer: IMuxer, protected val endpoint: IEndpoint, initialOnErrorListener: OnErrorListener? = null @@ -78,6 +76,9 @@ abstract class BaseStreamer( protected var videoConfig: VideoConfig? = null private var audioConfig: AudioConfig? = null + private val sourceOrientationProvider: ISourceOrientationProvider? + get() = videoSource?.orientationProvider + // Only handle stream error (error on muxer, endpoint,...) /** * Internal usage only @@ -90,7 +91,7 @@ abstract class BaseStreamer( private val audioEncoderListener = object : IEncoderListener { override fun onInputFrame(buffer: ByteBuffer): Frame { - return audioCapture!!.getFrame(buffer) + return audioSource!!.getFrame(buffer) } override fun onOutputFrame(frame: Frame) { @@ -106,15 +107,15 @@ abstract class BaseStreamer( private val videoEncoderListener = object : IEncoderListener { override fun onInputFrame(buffer: ByteBuffer): Frame { - return videoCapture!!.getFrame(buffer) + return videoSource!!.getFrame(buffer) } override fun onOutputFrame(frame: Frame) { videoStreamId?.let { try { - frame.pts += videoCapture!!.timestampOffset + frame.pts += videoSource!!.timestampOffset frame.dts = if (frame.dts != null) { - frame.dts!! + videoCapture.timestampOffset + frame.dts!! + videoSource.timestampOffset } else { null } @@ -154,30 +155,30 @@ abstract class BaseStreamer( } } - protected var audioEncoder = if (audioCapture != null) { + protected var audioEncoder = if (audioSource != null) { AudioMediaCodecEncoder(audioEncoderListener, onInternalErrorListener) } else { null } - protected var videoEncoder = if (videoCapture != null) { + protected var videoEncoder = if (videoSource != null) { VideoMediaCodecEncoder( videoEncoderListener, onInternalErrorListener, - videoCapture.hasSurface, - orientationProvider + videoSource.hasSurface, + sourceOrientationProvider ) } else { null } - override val settings = BaseStreamerSettings(audioCapture, audioEncoder, videoEncoder) + override val settings = BaseStreamerSettings(audioSource, audioEncoder, videoEncoder) private val hasAudio: Boolean - get() = audioCapture != null + get() = audioSource != null private val hasVideo: Boolean - get() = videoCapture != null + get() = videoSource != null init { - muxer.orientationProvider = orientationProvider + muxer.sourceOrientationProvider = sourceOrientationProvider muxer.listener = muxListener } @@ -200,7 +201,7 @@ abstract class BaseStreamer( this.audioConfig = audioConfig try { - audioCapture?.configure(audioConfig) + audioSource?.configure(audioConfig) audioEncoder?.release() audioEncoder?.configure(audioConfig) @@ -232,7 +233,7 @@ abstract class BaseStreamer( this.videoConfig = videoConfig try { - videoCapture?.configure(videoConfig) + videoSource?.configure(videoConfig) videoEncoder?.release() videoEncoder?.configure(videoConfig) @@ -292,10 +293,10 @@ abstract class BaseStreamer( muxer.startStream() - audioCapture?.startStream() + audioSource?.startStream() audioEncoder?.startStream() - videoCapture?.startStream() + videoSource?.startStream() videoEncoder?.startStream() } catch (e: Exception) { stopStream() @@ -325,10 +326,10 @@ abstract class BaseStreamer( * @see [stopStream] */ private fun stopStreamImpl() { - videoCapture?.stopStream() + videoSource?.stopStream() videoEncoder?.stopStream() audioEncoder?.stopStream() - audioCapture?.stopStream() + audioSource?.stopStream() muxer.stopStream() @@ -361,7 +362,7 @@ abstract class BaseStreamer( videoConfig?.let { videoEncoder?.configure(it) } - videoCapture?.encoderSurface = videoEncoder?.inputSurface + videoSource?.encoderSurface = videoEncoder?.inputSurface } /** @@ -374,8 +375,8 @@ abstract class BaseStreamer( audioEncoder?.release() videoEncoder?.codecSurface?.dispose() videoEncoder?.release() - audioCapture?.release() - videoCapture?.release() + audioSource?.release() + videoSource?.release() muxer.release() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/AudioOnlyFlvFileStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/AudioOnlyFlvFileStreamer.kt index f77eeaf2d..7940fbdfe 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/AudioOnlyFlvFileStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/AudioOnlyFlvFileStreamer.kt @@ -31,6 +31,6 @@ class AudioOnlyFlvFileStreamer( initialOnErrorListener: OnErrorListener? = null ) : BaseAudioOnlyFileStreamer( context = context, - muxer = FlvMuxer(context = context, writeToFile = true), + muxer = FlvMuxer(writeToFile = true), initialOnErrorListener = initialOnErrorListener ) \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/CameraFlvFileStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/CameraFlvFileStreamer.kt index 38760c102..2d8d3cd83 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/CameraFlvFileStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/file/CameraFlvFileStreamer.kt @@ -33,7 +33,7 @@ class CameraFlvFileStreamer( initialOnErrorListener: OnErrorListener? = null ) : BaseCameraFileStreamer( context = context, - muxer = FlvMuxer(context = context, writeToFile = true), + muxer = FlvMuxer(writeToFile = true), enableAudio = enableAudio, initialOnErrorListener = initialOnErrorListener ) \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/helpers/CameraStreamerConfigurationHelper.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/helpers/CameraStreamerConfigurationHelper.kt index 57dceaea2..d4cdd0378 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/helpers/CameraStreamerConfigurationHelper.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/helpers/CameraStreamerConfigurationHelper.kt @@ -23,9 +23,9 @@ import io.github.thibaultbee.streampack.internal.muxers.IVideoMuxerHelper import io.github.thibaultbee.streampack.internal.muxers.flv.FlvMuxerHelper import io.github.thibaultbee.streampack.internal.muxers.mp4.MP4MuxerHelper import io.github.thibaultbee.streampack.internal.muxers.ts.TSMuxerHelper -import io.github.thibaultbee.streampack.internal.sources.camera.getCameraFpsList -import io.github.thibaultbee.streampack.internal.sources.camera.getCameraOutputStreamSizes import io.github.thibaultbee.streampack.streamers.bases.BaseCameraStreamer +import io.github.thibaultbee.streampack.utils.getCameraFpsList +import io.github.thibaultbee.streampack.utils.getCameraOutputStreamSizes /** * Configuration helper for [BaseCameraStreamer]. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt index 2c44816b5..932e27cb8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt @@ -17,8 +17,8 @@ package io.github.thibaultbee.streampack.streamers.settings import io.github.thibaultbee.streampack.internal.encoders.AudioMediaCodecEncoder import io.github.thibaultbee.streampack.internal.encoders.VideoMediaCodecEncoder -import io.github.thibaultbee.streampack.internal.sources.IAudioCapture -import io.github.thibaultbee.streampack.internal.sources.camera.CameraCapture +import io.github.thibaultbee.streampack.internal.sources.IAudioSource +import io.github.thibaultbee.streampack.internal.sources.camera.CameraSource import io.github.thibaultbee.streampack.streamers.bases.BaseCameraStreamer import io.github.thibaultbee.streampack.streamers.interfaces.settings.IBaseCameraStreamerSettings import io.github.thibaultbee.streampack.utils.CameraSettings @@ -28,14 +28,14 @@ import io.github.thibaultbee.streampack.utils.CameraSettings * Get the base camera settings ie all settings available for [BaseCameraStreamer]. */ class BaseCameraStreamerSettings( - audioCapture: IAudioCapture?, - private val cameraCapture: CameraCapture, + audioSource: IAudioSource?, + private val cameraSource: CameraSource, audioEncoder: AudioMediaCodecEncoder?, videoEncoder: VideoMediaCodecEncoder? -) : BaseStreamerSettings(audioCapture, audioEncoder, videoEncoder), IBaseCameraStreamerSettings { +) : BaseStreamerSettings(audioSource, audioEncoder, videoEncoder), IBaseCameraStreamerSettings { /** * Get the camera settings (focus, zoom,...). */ override val camera: CameraSettings - get() = cameraCapture.settings + get() = cameraSource.settings } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt index 70fa8d678..bd7d28f9e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt @@ -17,20 +17,20 @@ package io.github.thibaultbee.streampack.streamers.settings import io.github.thibaultbee.streampack.internal.encoders.AudioMediaCodecEncoder import io.github.thibaultbee.streampack.internal.encoders.VideoMediaCodecEncoder -import io.github.thibaultbee.streampack.internal.sources.IAudioCapture +import io.github.thibaultbee.streampack.internal.sources.IAudioSource import io.github.thibaultbee.streampack.streamers.interfaces.settings.IAudioSettings import io.github.thibaultbee.streampack.streamers.interfaces.settings.IBaseStreamerSettings import io.github.thibaultbee.streampack.streamers.interfaces.settings.IVideoSettings open class BaseStreamerSettings( - audioCapture: IAudioCapture?, + audioSource: IAudioSource?, audioEncoder: AudioMediaCodecEncoder?, videoEncoder: VideoMediaCodecEncoder? ) : IBaseStreamerSettings { /** * Get audio settings */ - override val audio = BaseStreamerAudioSettings(audioCapture, audioEncoder) + override val audio = BaseStreamerAudioSettings(audioSource, audioEncoder) /** * Get video settings @@ -58,7 +58,7 @@ class BaseStreamerVideoSettings(private val videoEncoder: VideoMediaCodecEncoder } class BaseStreamerAudioSettings( - private val audioCapture: IAudioCapture?, + private val audioSource: IAudioSource?, private val audioEncoder: AudioMediaCodecEncoder? ) : IAudioSettings { @@ -85,11 +85,11 @@ class BaseStreamerAudioSettings( /** * @return [Boolean.true] if audio is muted, [Boolean.false] if audio is running. */ - get() = audioCapture?.isMuted ?: true + get() = audioSource?.isMuted ?: true /** * @param value [Boolean.true] to mute audio, [Boolean.false]to unmute audio. */ set(value) { - audioCapture?.isMuted = value + audioSource?.isMuted = value } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/utils/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/utils/CameraSettings.kt index ab8fda9ab..1ded12357 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/utils/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/utils/CameraSettings.kt @@ -29,6 +29,7 @@ import android.util.Rational import io.github.thibaultbee.streampack.internal.sources.camera.CameraController import io.github.thibaultbee.streampack.internal.utils.* import io.github.thibaultbee.streampack.internal.utils.extensions.clamp +import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait import io.github.thibaultbee.streampack.internal.utils.extensions.isNormalized import io.github.thibaultbee.streampack.internal.utils.extensions.normalize import io.github.thibaultbee.streampack.internal.utils.extensions.rotate @@ -43,7 +44,7 @@ import java.util.concurrent.TimeUnit * Use to change camera settings. * This object is returned by [BaseCameraStreamer.settings.camera]. */ -class CameraSettings(context: Context, private val cameraController: CameraController) { +class CameraSettings(context: Context, cameraController: CameraController) { /** * Current camera flash API. */ @@ -755,6 +756,53 @@ class FocusMetering( autoCancelHandle = null } + + /** + * Computes rotation required to transform the camera sensor output orientation to the + * device's current orientation in degrees. + * + * @param cameraId The camera to query for the sensor orientation. + * @param surfaceRotationDegrees The current Surface orientation in degrees. + * @return Relative rotation of the camera sensor output. + */ + private fun getSensorRotationDegrees( + context: Context, + cameraId: String, + surfaceRotationDegrees: Int = 0 + ): Int { + val characteristics = context.getCameraCharacteristics(cameraId) + val sensorOrientationDegrees = + characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) + + require(sensorOrientationDegrees != null) { + "Camera $cameraId has no defined sensor orientation." + } + + // Reverse device orientation for back-facing cameras. + val isFacingFront = characteristics.get(CameraCharacteristics.LENS_FACING) == + CameraCharacteristics.LENS_FACING_FRONT + + // Calculate desired orientation relative to camera orientation to make + // the image upright relative to the device orientation. + return getRelativeRotationDegrees( + sensorOrientationDegrees, + surfaceRotationDegrees, + isFacingFront + ) + } + + private fun getRelativeRotationDegrees( + sourceRotationDegrees: Int, + destRotationDegrees: Int, + isFacingFront: Boolean + ): Int { + return if (isFacingFront) { + (sourceRotationDegrees + destRotationDegrees + 360) % 360 + } else { + (sourceRotationDegrees - destRotationDegrees + 360) % 360 + } + } + /** * Sets the focus on tap. * @@ -764,7 +812,7 @@ class FocusMetering( */ fun onTap(point: PointF, fovRect: Rect, fovRotationDegree: Int) { val cameraId = cameraController.cameraId ?: throw IllegalStateException("Camera ID is null") - val relativeRotation = context.computeRelativeRotation(cameraId, fovRotationDegree) + val relativeRotation = getSensorRotationDegrees(context, cameraId, fovRotationDegree) var normalizedPoint = point.normalize(fovRect) normalizedPoint = normalizedPoint.rotate(relativeRotation) @@ -773,7 +821,7 @@ class FocusMetering( listOf(normalizedPoint), listOf(normalizedPoint), emptyList(), - if (OrientationUtils.isPortrait(relativeRotation)) { + if (context.isDevicePortrait) { Rational(fovRect.height(), fovRect.width()) } else { Rational(fovRect.width(), fovRect.height()) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/utils/CameraExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/utils/ContextCameraExtensions.kt similarity index 87% rename from core/src/main/java/io/github/thibaultbee/streampack/utils/CameraExtensions.kt rename to core/src/main/java/io/github/thibaultbee/streampack/utils/ContextCameraExtensions.kt index 69d922297..825d58e00 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/utils/CameraExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/utils/ContextCameraExtensions.kt @@ -16,13 +16,13 @@ package io.github.thibaultbee.streampack.utils import android.content.Context +import android.graphics.ImageFormat import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureResult import android.os.Build import android.util.Range import android.util.Size -import io.github.thibaultbee.streampack.internal.sources.camera.getCameraFpsList /** * Get camera characteristics. @@ -297,36 +297,44 @@ fun Context.isOpticalStabilizationAvailable(cameraId: String) = CaptureResult.LENS_OPTICAL_STABILIZATION_MODE_ON ) ?: false + /** - * Computes rotation required to transform the camera sensor output orientation to the - * device's current orientation in degrees. + * Gets all output capture sizes. * - * @param cameraId The camera to query for the sensor orientation. - * @param surfaceRotationDegrees The current device orientation as a Surface constant. - * @return Relative rotation of the camera sensor output. + * @return List of resolutions supported by all camera */ -fun Context.computeRelativeRotation( - cameraId: String, - surfaceRotationDegrees: Int -): Int { - val characteristics = getCameraCharacteristics(cameraId) - val sensorOrientationDegrees = - characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) - - // Reverse device orientation for back-facing cameras. - val sign = if (characteristics.get(CameraCharacteristics.LENS_FACING) == - CameraCharacteristics.LENS_FACING_FRONT - ) { - 1 - } else if (characteristics.get(CameraCharacteristics.LENS_FACING) == - CameraCharacteristics.LENS_FACING_BACK - ) { - -1 - } else { - throw IllegalStateException("Unknown lens facing") +fun Context.getCameraOutputStreamSizes(): List { + val cameraIdList = cameraList + val resolutionSet = mutableSetOf() + cameraIdList.forEach { cameraId -> + resolutionSet.addAll(getCameraOutputStreamSizes(cameraId)) } + return resolutionSet.toList() +} - // Calculate desired orientation relative to camera orientation to make - // the image upright relative to the device orientation. - return (sensorOrientationDegrees!! - surfaceRotationDegrees * sign + 360) % 360 +/** + * Gets list of output stream sizes of a camera. + * + * @param cameraId camera id + * @return List of resolutions supported by a camera + * @see [Context.getCameraOutputStreamSizes] + */ +fun Context.getCameraOutputStreamSizes( + cameraId: String, + imageFormat: Int = ImageFormat.YUV_420_888 +): List { + return this.getCameraCharacteristics(cameraId)[CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP]?.getOutputSizes( + imageFormat + )?.toList() ?: emptyList() } + +/** + * Get list of framerate for a camera. + * + * @param cameraId camera id + * @return List of fps supported by a camera + */ +fun Context.getCameraFpsList(cameraId: String): List> { + return this.getCameraCharacteristics(cameraId)[CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES]?.toList() + ?: emptyList() +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/utils/OrientationUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/utils/OrientationUtils.kt index bbc586aa6..246874581 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/utils/OrientationUtils.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/utils/OrientationUtils.kt @@ -18,26 +18,16 @@ package io.github.thibaultbee.streampack.utils import android.view.Surface object OrientationUtils { - fun getSurfaceOrientation(surfaceOrientation: Int): Int { + /** + * Returns the surface orientation in degrees from [Surface] orientation ([Surface.ROTATION_0], ...). + */ + fun getSurfaceOrientationDegrees(surfaceOrientation: Int): Int { return when (surfaceOrientation) { Surface.ROTATION_0 -> 0 Surface.ROTATION_90 -> 90 Surface.ROTATION_180 -> 180 Surface.ROTATION_270 -> 270 - else -> 0 - } - } - - /** - * Returns true if the rotation degrees is 90 or 270. - */ - fun isPortrait(rotationDegrees: Int): Boolean { - if (rotationDegrees == 90 || rotationDegrees == 270) { - return true - } - if (rotationDegrees == 0 || rotationDegrees == 180) { - return false + else -> throw IllegalArgumentException("Invalid surface orientation: $surfaceOrientation") } - throw IllegalArgumentException("Invalid rotation degrees: $rotationDegrees") } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt b/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt index 615667d8c..ef24fb522 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt @@ -145,7 +145,7 @@ class PreviewView @JvmOverloads constructor( it.settings.camera.focusMetering.onTap( PointF(x, y), Rect(this.x.toInt(), this.y.toInt(), width, height), - OrientationUtils.getSurfaceOrientation(display.rotation) + OrientationUtils.getSurfaceOrientationDegrees(display.rotation) ) } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadataTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadataTest.kt index bb203782c..16350e7e5 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadataTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadataTest.kt @@ -17,7 +17,6 @@ package io.github.thibaultbee.streampack.internal.muxers.flv.tags import android.util.Size import io.github.thibaultbee.streampack.data.VideoConfig -import io.github.thibaultbee.streampack.internal.data.orientation.FixedOrientationProvider import io.github.thibaultbee.streampack.internal.utils.extensions.toByteArray import io.github.thibaultbee.streampack.utils.MockUtils import org.junit.Assert.assertArrayEquals @@ -185,8 +184,7 @@ class OnMetadataTest { ) MockUtils.mockSizeConstructor(640, 480) - val onMetadata = OnMetadata( - FixedOrientationProvider(0), + val onMetadata = OnMetadata.fromConfigs( listOf( VideoConfig(resolution = Size(640, 480), profile = 0, level = 0) ) diff --git a/core/src/test/java/io/github/thibaultbee/streampack/internal/sources/AudioCaptureUnitTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/internal/sources/AudioCaptureUnitTest.kt index 17bd00f01..bb8d74d7d 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/internal/sources/AudioCaptureUnitTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/internal/sources/AudioCaptureUnitTest.kt @@ -21,21 +21,21 @@ import org.junit.Assert import org.junit.Test import java.nio.ByteBuffer -class AudioCaptureUnitTest { +class AudioSourceUnitTest { init { Logger.logger = FakeLogger() } @Test fun `assert exception on bad state`() { - val audioCapture = AudioCapture() + val audioSource = AudioSource() try { - audioCapture.startStream() + audioSource.startStream() Assert.fail() } catch (_: Exception) { } try { - audioCapture.getFrame(ByteBuffer.allocate(10)) + audioSource.getFrame(ByteBuffer.allocate(10)) Assert.fail() } catch (_: Exception) { } @@ -43,14 +43,14 @@ class AudioCaptureUnitTest { @Test fun `assert no exception on bad state`() { - val audioCapture = AudioCapture() + val audioSource = AudioSource() try { - audioCapture.stopStream() + audioSource.stopStream() } catch (e: Exception) { Assert.fail() } try { - audioCapture.release() + audioSource.release() } catch (e: Exception) { Assert.fail() } diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/AudioOnlyRtmpLiveStreamer.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/AudioOnlyRtmpLiveStreamer.kt index 064ddf44b..8bf7e4651 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/AudioOnlyRtmpLiveStreamer.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/AudioOnlyRtmpLiveStreamer.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.ext.rtmp.streamers import android.content.Context import io.github.thibaultbee.streampack.ext.rtmp.internal.endpoints.RtmpProducer import io.github.thibaultbee.streampack.internal.muxers.flv.FlvMuxer -import io.github.thibaultbee.streampack.internal.muxers.flv.tags.video.ExtendedVideoTag import io.github.thibaultbee.streampack.listeners.OnConnectionListener import io.github.thibaultbee.streampack.listeners.OnErrorListener import io.github.thibaultbee.streampack.streamers.live.BaseAudioOnlyLiveStreamer @@ -36,7 +35,7 @@ class AudioOnlyRtmpLiveStreamer( initialOnConnectionListener: OnConnectionListener? = null ) : BaseAudioOnlyLiveStreamer( context = context, - muxer = FlvMuxer(context = context, writeToFile = false), + muxer = FlvMuxer(writeToFile = false), endpoint = RtmpProducer(hasAudio = true, hasVideo = false), initialOnErrorListener = initialOnErrorListener, initialOnConnectionListener = initialOnConnectionListener diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/CameraRtmpLiveStreamer.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/CameraRtmpLiveStreamer.kt index 0d362c8ed..8b50fd725 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/CameraRtmpLiveStreamer.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/CameraRtmpLiveStreamer.kt @@ -39,7 +39,7 @@ class CameraRtmpLiveStreamer( ) : BaseCameraLiveStreamer( context = context, enableAudio = enableAudio, - muxer = FlvMuxer(context = context, writeToFile = false), + muxer = FlvMuxer(writeToFile = false), endpoint = RtmpProducer(hasAudio = enableAudio, hasVideo = true), initialOnErrorListener = initialOnErrorListener, initialOnConnectionListener = initialOnConnectionListener diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/ScreenRecorderRtmpLiveStreamer.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/ScreenRecorderRtmpLiveStreamer.kt index 5d162886f..8dc4fd193 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/ScreenRecorderRtmpLiveStreamer.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/streamers/ScreenRecorderRtmpLiveStreamer.kt @@ -43,7 +43,7 @@ class ScreenRecorderRtmpLiveStreamer( ) : BaseScreenRecorderLiveStreamer( context = context, enableAudio = enableAudio, - muxer = FlvMuxer(context = context, writeToFile = false), + muxer = FlvMuxer(writeToFile = false), endpoint = RtmpProducer(hasAudio = enableAudio, hasVideo = true), initialOnErrorListener = initialOnErrorListener, initialOnConnectionListener = initialOnConnectionListener