From 7b1bb4e47595f85d5f421808144e27e9ab0272f4 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 8 Jun 2021 17:16:57 +0200 Subject: [PATCH] feat(*): Add support for camera with Realtime source. See https://github.com/ThibaultBee/StreamPack/issues/12 --- .../internal/sources/camera/CameraCapture.kt | 6 ++ .../internal/sources/camera/CameraHelper.kt | 72 +++++++++++++++++++ .../streamers/BaseCaptureStreamer.kt | 6 ++ 3 files changed, 84 insertions(+) create mode 100644 sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraHelper.kt diff --git a/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt b/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt index b4811c392..c13473de4 100644 --- a/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt +++ b/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraCapture.kt @@ -44,6 +44,12 @@ class CameraCapture( private var cameraController = CameraController(context, logger = logger) var settings = CameraSettings(cameraController) + /** + * As timestamp source differs from one camera to another. Computes an offset. + */ + val timestampOffset = CameraHelper.getTimeOffsetToMonoClock(context, cameraId) + + private var fps: Int = 30 private var isStreaming = false internal var isPreviewing = false diff --git a/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraHelper.kt b/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraHelper.kt new file mode 100644 index 000000000..d3368afd9 --- /dev/null +++ b/sdk/src/main/java/com/github/thibaultbee/streampack/internal/sources/camera/CameraHelper.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2019 The MediaPipe Authors. + * Copyright (C) 2021 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.thibaultbee.streampack.internal.sources.camera + +import android.content.Context +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraMetadata +import android.os.SystemClock +import com.github.thibaultbee.streampack.utils.getCameraCharacteristics + + +object CameraHelper { + // Number of attempts for calculating the offset between the camera's clock and MONOTONIC clock. + private const val CLOCK_OFFSET_CALIBRATION_ATTEMPTS = 3 + + // Computes the difference between the camera's clock and MONOTONIC clock using camera's + // timestamp source information. This function assumes by default that the camera timestamp + // source is aligned to CLOCK_MONOTONIC. This is useful when the camera is being used + // synchronously with other sensors that yield timestamps in the MONOTONIC timebase, such as + // AudioRecord for audio data. The offset is returned in milliseconds. + fun getTimeOffsetToMonoClock(context: Context, cameraId: String): Long { + return if (context.getCameraCharacteristics(cameraId) + .get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraMetadata.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME + ) { + // This clock shares the same timebase as SystemClock.elapsedRealtimeNanos(), see + // https://developer.android.com/reference/android/hardware/camera2/CameraMetadata.html#SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME. + getOffsetFromRealtimeTimestampSource() + } else { + getOffsetFromUnknownTimestampSource() + } + } + + private fun getOffsetFromUnknownTimestampSource(): Long { + // Implementation-wise, this timestamp source has the same timebase as CLOCK_MONOTONIC, see + // https://stackoverflow.com/questions/38585761/what-is-the-timebase-of-the-timestamp-of-cameradevice. + return 0L + } + + private fun getOffsetFromRealtimeTimestampSource(): Long { + // Measure the offset of the REALTIME clock w.r.t. the MONOTONIC clock. Do + // CLOCK_OFFSET_CALIBRATION_ATTEMPTS measurements and choose the offset computed with the + // smallest delay between measurements. When the camera returns a timestamp ts, the + // timestamp in MONOTONIC timebase will now be (ts + cameraTimeOffsetToMonoClock). + var offset = Long.MAX_VALUE + var lowestGap = Long.MAX_VALUE + for (i in 0 until CLOCK_OFFSET_CALIBRATION_ATTEMPTS) { + val startMonoTs = System.nanoTime() + val realTs = SystemClock.elapsedRealtimeNanos() + val endMonoTs = System.nanoTime() + val gapMonoTs = endMonoTs - startMonoTs + if (gapMonoTs < lowestGap) { + lowestGap = gapMonoTs + offset = (startMonoTs + endMonoTs) / 2 - realTs + } + } + return offset / 1000 + } +} \ No newline at end of file diff --git a/sdk/src/main/java/com/github/thibaultbee/streampack/streamers/BaseCaptureStreamer.kt b/sdk/src/main/java/com/github/thibaultbee/streampack/streamers/BaseCaptureStreamer.kt index aa53c5225..490920d78 100644 --- a/sdk/src/main/java/com/github/thibaultbee/streampack/streamers/BaseCaptureStreamer.kt +++ b/sdk/src/main/java/com/github/thibaultbee/streampack/streamers/BaseCaptureStreamer.kt @@ -135,6 +135,12 @@ open class BaseCaptureStreamer( override fun onOutputFrame(frame: Frame) { videoTsStreamId?.let { try { + frame.pts += videoSource.timestampOffset + frame.dts = if (frame.dts != null) { + frame.dts!! + videoSource.timestampOffset + } else { + null + } tsMux.encode(frame, it) } catch (e: Exception) { // Send exception to encoder